Dependency update
This commit is contained in:
+7
-7
@@ -153,7 +153,7 @@ type Options struct {
|
||||
BlockCacher Cacher
|
||||
|
||||
// BlockCacheCapacity defines the capacity of the 'sorted table' block caching.
|
||||
// Use -1 for zero, this has same effect with specifying NoCacher to BlockCacher.
|
||||
// Use -1 for zero, this has same effect as specifying NoCacher to BlockCacher.
|
||||
//
|
||||
// The default value is 8MiB.
|
||||
BlockCacheCapacity int
|
||||
@@ -308,7 +308,7 @@ type Options struct {
|
||||
OpenFilesCacher Cacher
|
||||
|
||||
// OpenFilesCacheCapacity defines the capacity of the open files caching.
|
||||
// Use -1 for zero, this has same effect with specifying NoCacher to OpenFilesCacher.
|
||||
// Use -1 for zero, this has same effect as specifying NoCacher to OpenFilesCacher.
|
||||
//
|
||||
// The default value is 500.
|
||||
OpenFilesCacheCapacity int
|
||||
@@ -355,9 +355,9 @@ func (o *Options) GetBlockCacher() Cacher {
|
||||
}
|
||||
|
||||
func (o *Options) GetBlockCacheCapacity() int {
|
||||
if o == nil || o.BlockCacheCapacity <= 0 {
|
||||
if o == nil || o.BlockCacheCapacity == 0 {
|
||||
return DefaultBlockCacheCapacity
|
||||
} else if o.BlockCacheCapacity == -1 {
|
||||
} else if o.BlockCacheCapacity < 0 {
|
||||
return 0
|
||||
}
|
||||
return o.BlockCacheCapacity
|
||||
@@ -497,7 +497,7 @@ func (o *Options) GetMaxMemCompationLevel() int {
|
||||
if o != nil {
|
||||
if o.MaxMemCompationLevel > 0 {
|
||||
level = o.MaxMemCompationLevel
|
||||
} else if o.MaxMemCompationLevel == -1 {
|
||||
} else if o.MaxMemCompationLevel < 0 {
|
||||
level = 0
|
||||
}
|
||||
}
|
||||
@@ -525,9 +525,9 @@ func (o *Options) GetOpenFilesCacher() Cacher {
|
||||
}
|
||||
|
||||
func (o *Options) GetOpenFilesCacheCapacity() int {
|
||||
if o == nil || o.OpenFilesCacheCapacity <= 0 {
|
||||
if o == nil || o.OpenFilesCacheCapacity == 0 {
|
||||
return DefaultOpenFilesCacheCapacity
|
||||
} else if o.OpenFilesCacheCapacity == -1 {
|
||||
} else if o.OpenFilesCacheCapacity < 0 {
|
||||
return 0
|
||||
}
|
||||
return o.OpenFilesCacheCapacity
|
||||
|
||||
+170
-2
@@ -7,10 +7,15 @@ package snappy
|
||||
import (
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"io"
|
||||
)
|
||||
|
||||
// ErrCorrupt reports that the input is invalid.
|
||||
var ErrCorrupt = errors.New("snappy: corrupt input")
|
||||
var (
|
||||
// ErrCorrupt reports that the input is invalid.
|
||||
ErrCorrupt = errors.New("snappy: corrupt input")
|
||||
// ErrUnsupported reports that the input isn't supported.
|
||||
ErrUnsupported = errors.New("snappy: unsupported input")
|
||||
)
|
||||
|
||||
// DecodedLen returns the length of the decoded block.
|
||||
func DecodedLen(src []byte) (int, error) {
|
||||
@@ -122,3 +127,166 @@ func Decode(dst, src []byte) ([]byte, error) {
|
||||
}
|
||||
return dst[:d], nil
|
||||
}
|
||||
|
||||
// NewReader returns a new Reader that decompresses from r, using the framing
|
||||
// format described at
|
||||
// https://code.google.com/p/snappy/source/browse/trunk/framing_format.txt
|
||||
func NewReader(r io.Reader) *Reader {
|
||||
return &Reader{
|
||||
r: r,
|
||||
decoded: make([]byte, maxUncompressedChunkLen),
|
||||
buf: make([]byte, MaxEncodedLen(maxUncompressedChunkLen)+checksumSize),
|
||||
}
|
||||
}
|
||||
|
||||
// Reader is an io.Reader than can read Snappy-compressed bytes.
|
||||
type Reader struct {
|
||||
r io.Reader
|
||||
err error
|
||||
decoded []byte
|
||||
buf []byte
|
||||
// decoded[i:j] contains decoded bytes that have not yet been passed on.
|
||||
i, j int
|
||||
readHeader bool
|
||||
}
|
||||
|
||||
// Reset discards any buffered data, resets all state, and switches the Snappy
|
||||
// reader to read from r. This permits reusing a Reader rather than allocating
|
||||
// a new one.
|
||||
func (r *Reader) Reset(reader io.Reader) {
|
||||
r.r = reader
|
||||
r.err = nil
|
||||
r.i = 0
|
||||
r.j = 0
|
||||
r.readHeader = false
|
||||
}
|
||||
|
||||
func (r *Reader) readFull(p []byte) (ok bool) {
|
||||
if _, r.err = io.ReadFull(r.r, p); r.err != nil {
|
||||
if r.err == io.ErrUnexpectedEOF {
|
||||
r.err = ErrCorrupt
|
||||
}
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// Read satisfies the io.Reader interface.
|
||||
func (r *Reader) Read(p []byte) (int, error) {
|
||||
if r.err != nil {
|
||||
return 0, r.err
|
||||
}
|
||||
for {
|
||||
if r.i < r.j {
|
||||
n := copy(p, r.decoded[r.i:r.j])
|
||||
r.i += n
|
||||
return n, nil
|
||||
}
|
||||
if !r.readFull(r.buf[:4]) {
|
||||
return 0, r.err
|
||||
}
|
||||
chunkType := r.buf[0]
|
||||
if !r.readHeader {
|
||||
if chunkType != chunkTypeStreamIdentifier {
|
||||
r.err = ErrCorrupt
|
||||
return 0, r.err
|
||||
}
|
||||
r.readHeader = true
|
||||
}
|
||||
chunkLen := int(r.buf[1]) | int(r.buf[2])<<8 | int(r.buf[3])<<16
|
||||
if chunkLen > len(r.buf) {
|
||||
r.err = ErrUnsupported
|
||||
return 0, r.err
|
||||
}
|
||||
|
||||
// The chunk types are specified at
|
||||
// https://code.google.com/p/snappy/source/browse/trunk/framing_format.txt
|
||||
switch chunkType {
|
||||
case chunkTypeCompressedData:
|
||||
// Section 4.2. Compressed data (chunk type 0x00).
|
||||
if chunkLen < checksumSize {
|
||||
r.err = ErrCorrupt
|
||||
return 0, r.err
|
||||
}
|
||||
buf := r.buf[:chunkLen]
|
||||
if !r.readFull(buf) {
|
||||
return 0, r.err
|
||||
}
|
||||
checksum := uint32(buf[0]) | uint32(buf[1])<<8 | uint32(buf[2])<<16 | uint32(buf[3])<<24
|
||||
buf = buf[checksumSize:]
|
||||
|
||||
n, err := DecodedLen(buf)
|
||||
if err != nil {
|
||||
r.err = err
|
||||
return 0, r.err
|
||||
}
|
||||
if n > len(r.decoded) {
|
||||
r.err = ErrCorrupt
|
||||
return 0, r.err
|
||||
}
|
||||
if _, err := Decode(r.decoded, buf); err != nil {
|
||||
r.err = err
|
||||
return 0, r.err
|
||||
}
|
||||
if crc(r.decoded[:n]) != checksum {
|
||||
r.err = ErrCorrupt
|
||||
return 0, r.err
|
||||
}
|
||||
r.i, r.j = 0, n
|
||||
continue
|
||||
|
||||
case chunkTypeUncompressedData:
|
||||
// Section 4.3. Uncompressed data (chunk type 0x01).
|
||||
if chunkLen < checksumSize {
|
||||
r.err = ErrCorrupt
|
||||
return 0, r.err
|
||||
}
|
||||
buf := r.buf[:checksumSize]
|
||||
if !r.readFull(buf) {
|
||||
return 0, r.err
|
||||
}
|
||||
checksum := uint32(buf[0]) | uint32(buf[1])<<8 | uint32(buf[2])<<16 | uint32(buf[3])<<24
|
||||
// Read directly into r.decoded instead of via r.buf.
|
||||
n := chunkLen - checksumSize
|
||||
if !r.readFull(r.decoded[:n]) {
|
||||
return 0, r.err
|
||||
}
|
||||
if crc(r.decoded[:n]) != checksum {
|
||||
r.err = ErrCorrupt
|
||||
return 0, r.err
|
||||
}
|
||||
r.i, r.j = 0, n
|
||||
continue
|
||||
|
||||
case chunkTypeStreamIdentifier:
|
||||
// Section 4.1. Stream identifier (chunk type 0xff).
|
||||
if chunkLen != len(magicBody) {
|
||||
r.err = ErrCorrupt
|
||||
return 0, r.err
|
||||
}
|
||||
if !r.readFull(r.buf[:len(magicBody)]) {
|
||||
return 0, r.err
|
||||
}
|
||||
for i := 0; i < len(magicBody); i++ {
|
||||
if r.buf[i] != magicBody[i] {
|
||||
r.err = ErrCorrupt
|
||||
return 0, r.err
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if chunkType <= 0x7f {
|
||||
// Section 4.5. Reserved unskippable chunks (chunk types 0x02-0x7f).
|
||||
r.err = ErrUnsupported
|
||||
return 0, r.err
|
||||
|
||||
} else {
|
||||
// Section 4.4 Padding (chunk type 0xfe).
|
||||
// Section 4.6. Reserved skippable chunks (chunk types 0x80-0xfd).
|
||||
if !r.readFull(r.buf[:chunkLen]) {
|
||||
return 0, r.err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+84
@@ -6,6 +6,7 @@ package snappy
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"io"
|
||||
)
|
||||
|
||||
// We limit how far copy back-references can go, the same as the C++ code.
|
||||
@@ -172,3 +173,86 @@ func MaxEncodedLen(srcLen int) int {
|
||||
// This last factor dominates the blowup, so the final estimate is:
|
||||
return 32 + srcLen + srcLen/6
|
||||
}
|
||||
|
||||
// NewWriter returns a new Writer that compresses to w, using the framing
|
||||
// format described at
|
||||
// https://code.google.com/p/snappy/source/browse/trunk/framing_format.txt
|
||||
func NewWriter(w io.Writer) *Writer {
|
||||
return &Writer{
|
||||
w: w,
|
||||
enc: make([]byte, MaxEncodedLen(maxUncompressedChunkLen)),
|
||||
}
|
||||
}
|
||||
|
||||
// Writer is an io.Writer than can write Snappy-compressed bytes.
|
||||
type Writer struct {
|
||||
w io.Writer
|
||||
err error
|
||||
enc []byte
|
||||
buf [checksumSize + chunkHeaderSize]byte
|
||||
wroteHeader bool
|
||||
}
|
||||
|
||||
// Reset discards the writer's state and switches the Snappy writer to write to
|
||||
// w. This permits reusing a Writer rather than allocating a new one.
|
||||
func (w *Writer) Reset(writer io.Writer) {
|
||||
w.w = writer
|
||||
w.err = nil
|
||||
w.wroteHeader = false
|
||||
}
|
||||
|
||||
// Write satisfies the io.Writer interface.
|
||||
func (w *Writer) Write(p []byte) (n int, errRet error) {
|
||||
if w.err != nil {
|
||||
return 0, w.err
|
||||
}
|
||||
if !w.wroteHeader {
|
||||
copy(w.enc, magicChunk)
|
||||
if _, err := w.w.Write(w.enc[:len(magicChunk)]); err != nil {
|
||||
w.err = err
|
||||
return n, err
|
||||
}
|
||||
w.wroteHeader = true
|
||||
}
|
||||
for len(p) > 0 {
|
||||
var uncompressed []byte
|
||||
if len(p) > maxUncompressedChunkLen {
|
||||
uncompressed, p = p[:maxUncompressedChunkLen], p[maxUncompressedChunkLen:]
|
||||
} else {
|
||||
uncompressed, p = p, nil
|
||||
}
|
||||
checksum := crc(uncompressed)
|
||||
|
||||
// Compress the buffer, discarding the result if the improvement
|
||||
// isn't at least 12.5%.
|
||||
chunkType := uint8(chunkTypeCompressedData)
|
||||
chunkBody, err := Encode(w.enc, uncompressed)
|
||||
if err != nil {
|
||||
w.err = err
|
||||
return n, err
|
||||
}
|
||||
if len(chunkBody) >= len(uncompressed)-len(uncompressed)/8 {
|
||||
chunkType, chunkBody = chunkTypeUncompressedData, uncompressed
|
||||
}
|
||||
|
||||
chunkLen := 4 + len(chunkBody)
|
||||
w.buf[0] = chunkType
|
||||
w.buf[1] = uint8(chunkLen >> 0)
|
||||
w.buf[2] = uint8(chunkLen >> 8)
|
||||
w.buf[3] = uint8(chunkLen >> 16)
|
||||
w.buf[4] = uint8(checksum >> 0)
|
||||
w.buf[5] = uint8(checksum >> 8)
|
||||
w.buf[6] = uint8(checksum >> 16)
|
||||
w.buf[7] = uint8(checksum >> 24)
|
||||
if _, err = w.w.Write(w.buf[:]); err != nil {
|
||||
w.err = err
|
||||
return n, err
|
||||
}
|
||||
if _, err = w.w.Write(chunkBody); err != nil {
|
||||
w.err = err
|
||||
return n, err
|
||||
}
|
||||
n += len(uncompressed)
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
+30
@@ -8,6 +8,10 @@
|
||||
// The C++ snappy implementation is at http://code.google.com/p/snappy/
|
||||
package snappy
|
||||
|
||||
import (
|
||||
"hash/crc32"
|
||||
)
|
||||
|
||||
/*
|
||||
Each encoded block begins with the varint-encoded length of the decoded data,
|
||||
followed by a sequence of chunks. Chunks begin and end on byte boundaries. The
|
||||
@@ -36,3 +40,29 @@ const (
|
||||
tagCopy2 = 0x02
|
||||
tagCopy4 = 0x03
|
||||
)
|
||||
|
||||
const (
|
||||
checksumSize = 4
|
||||
chunkHeaderSize = 4
|
||||
magicChunk = "\xff\x06\x00\x00" + magicBody
|
||||
magicBody = "sNaPpY"
|
||||
// https://code.google.com/p/snappy/source/browse/trunk/framing_format.txt says
|
||||
// that "the uncompressed data in a chunk must be no longer than 65536 bytes".
|
||||
maxUncompressedChunkLen = 65536
|
||||
)
|
||||
|
||||
const (
|
||||
chunkTypeCompressedData = 0x00
|
||||
chunkTypeUncompressedData = 0x01
|
||||
chunkTypePadding = 0xfe
|
||||
chunkTypeStreamIdentifier = 0xff
|
||||
)
|
||||
|
||||
var crcTable = crc32.MakeTable(crc32.Castagnoli)
|
||||
|
||||
// crc implements the checksum specified in section 3 of
|
||||
// https://code.google.com/p/snappy/source/browse/trunk/framing_format.txt
|
||||
func crc(b []byte) uint32 {
|
||||
c := crc32.Update(0, crcTable, b)
|
||||
return uint32(c>>15|c<<17) + 0xa282ead8
|
||||
}
|
||||
|
||||
+152
-49
@@ -18,7 +18,10 @@ import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
var download = flag.Bool("download", false, "If true, download any missing files before running benchmarks")
|
||||
var (
|
||||
download = flag.Bool("download", false, "If true, download any missing files before running benchmarks")
|
||||
testdata = flag.String("testdata", "testdata", "Directory containing the test data")
|
||||
)
|
||||
|
||||
func roundtrip(b, ebuf, dbuf []byte) error {
|
||||
e, err := Encode(ebuf, b)
|
||||
@@ -55,11 +58,11 @@ func TestSmallCopy(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestSmallRand(t *testing.T) {
|
||||
rand.Seed(27354294)
|
||||
rng := rand.New(rand.NewSource(27354294))
|
||||
for n := 1; n < 20000; n += 23 {
|
||||
b := make([]byte, n)
|
||||
for i, _ := range b {
|
||||
b[i] = uint8(rand.Uint32())
|
||||
for i := range b {
|
||||
b[i] = uint8(rng.Uint32())
|
||||
}
|
||||
if err := roundtrip(b, nil, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -70,7 +73,7 @@ func TestSmallRand(t *testing.T) {
|
||||
func TestSmallRegular(t *testing.T) {
|
||||
for n := 1; n < 20000; n += 23 {
|
||||
b := make([]byte, n)
|
||||
for i, _ := range b {
|
||||
for i := range b {
|
||||
b[i] = uint8(i%10 + 'a')
|
||||
}
|
||||
if err := roundtrip(b, nil, nil); err != nil {
|
||||
@@ -79,6 +82,120 @@ func TestSmallRegular(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func cmp(a, b []byte) error {
|
||||
if len(a) != len(b) {
|
||||
return fmt.Errorf("got %d bytes, want %d", len(a), len(b))
|
||||
}
|
||||
for i := range a {
|
||||
if a[i] != b[i] {
|
||||
return fmt.Errorf("byte #%d: got 0x%02x, want 0x%02x", i, a[i], b[i])
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestFramingFormat(t *testing.T) {
|
||||
// src is comprised of alternating 1e5-sized sequences of random
|
||||
// (incompressible) bytes and repeated (compressible) bytes. 1e5 was chosen
|
||||
// because it is larger than maxUncompressedChunkLen (64k).
|
||||
src := make([]byte, 1e6)
|
||||
rng := rand.New(rand.NewSource(1))
|
||||
for i := 0; i < 10; i++ {
|
||||
if i%2 == 0 {
|
||||
for j := 0; j < 1e5; j++ {
|
||||
src[1e5*i+j] = uint8(rng.Intn(256))
|
||||
}
|
||||
} else {
|
||||
for j := 0; j < 1e5; j++ {
|
||||
src[1e5*i+j] = uint8(i)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
buf := new(bytes.Buffer)
|
||||
if _, err := NewWriter(buf).Write(src); err != nil {
|
||||
t.Fatalf("Write: encoding: %v", err)
|
||||
}
|
||||
dst, err := ioutil.ReadAll(NewReader(buf))
|
||||
if err != nil {
|
||||
t.Fatalf("ReadAll: decoding: %v", err)
|
||||
}
|
||||
if err := cmp(dst, src); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReaderReset(t *testing.T) {
|
||||
gold := bytes.Repeat([]byte("All that is gold does not glitter,\n"), 10000)
|
||||
buf := new(bytes.Buffer)
|
||||
if _, err := NewWriter(buf).Write(gold); err != nil {
|
||||
t.Fatalf("Write: %v", err)
|
||||
}
|
||||
encoded, invalid, partial := buf.String(), "invalid", "partial"
|
||||
r := NewReader(nil)
|
||||
for i, s := range []string{encoded, invalid, partial, encoded, partial, invalid, encoded, encoded} {
|
||||
if s == partial {
|
||||
r.Reset(strings.NewReader(encoded))
|
||||
if _, err := r.Read(make([]byte, 101)); err != nil {
|
||||
t.Errorf("#%d: %v", i, err)
|
||||
continue
|
||||
}
|
||||
continue
|
||||
}
|
||||
r.Reset(strings.NewReader(s))
|
||||
got, err := ioutil.ReadAll(r)
|
||||
switch s {
|
||||
case encoded:
|
||||
if err != nil {
|
||||
t.Errorf("#%d: %v", i, err)
|
||||
continue
|
||||
}
|
||||
if err := cmp(got, gold); err != nil {
|
||||
t.Errorf("#%d: %v", i, err)
|
||||
continue
|
||||
}
|
||||
case invalid:
|
||||
if err == nil {
|
||||
t.Errorf("#%d: got nil error, want non-nil", i)
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriterReset(t *testing.T) {
|
||||
gold := bytes.Repeat([]byte("Not all those who wander are lost;\n"), 10000)
|
||||
var gots, wants [][]byte
|
||||
const n = 20
|
||||
w, failed := NewWriter(nil), false
|
||||
for i := 0; i <= n; i++ {
|
||||
buf := new(bytes.Buffer)
|
||||
w.Reset(buf)
|
||||
want := gold[:len(gold)*i/n]
|
||||
if _, err := w.Write(want); err != nil {
|
||||
t.Errorf("#%d: Write: %v", i, err)
|
||||
failed = true
|
||||
continue
|
||||
}
|
||||
got, err := ioutil.ReadAll(NewReader(buf))
|
||||
if err != nil {
|
||||
t.Errorf("#%d: ReadAll: %v", i, err)
|
||||
failed = true
|
||||
continue
|
||||
}
|
||||
gots = append(gots, got)
|
||||
wants = append(wants, want)
|
||||
}
|
||||
if failed {
|
||||
return
|
||||
}
|
||||
for i := range gots {
|
||||
if err := cmp(gots[i], wants[i]); err != nil {
|
||||
t.Errorf("#%d: %v", i, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func benchDecode(b *testing.B, src []byte) {
|
||||
encoded, err := Encode(nil, src)
|
||||
if err != nil {
|
||||
@@ -102,7 +219,7 @@ func benchEncode(b *testing.B, src []byte) {
|
||||
}
|
||||
}
|
||||
|
||||
func readFile(b *testing.B, filename string) []byte {
|
||||
func readFile(b testing.TB, filename string) []byte {
|
||||
src, err := ioutil.ReadFile(filename)
|
||||
if err != nil {
|
||||
b.Fatalf("failed reading %s: %s", filename, err)
|
||||
@@ -144,7 +261,7 @@ func BenchmarkWordsEncode1e5(b *testing.B) { benchWords(b, 1e5, false) }
|
||||
func BenchmarkWordsEncode1e6(b *testing.B) { benchWords(b, 1e6, false) }
|
||||
|
||||
// testFiles' values are copied directly from
|
||||
// https://code.google.com/p/snappy/source/browse/trunk/snappy_unittest.cc.
|
||||
// https://raw.githubusercontent.com/google/snappy/master/snappy_unittest.cc
|
||||
// The label field is unused in snappy-go.
|
||||
var testFiles = []struct {
|
||||
label string
|
||||
@@ -152,29 +269,36 @@ var testFiles = []struct {
|
||||
}{
|
||||
{"html", "html"},
|
||||
{"urls", "urls.10K"},
|
||||
{"jpg", "house.jpg"},
|
||||
{"pdf", "mapreduce-osdi-1.pdf"},
|
||||
{"jpg", "fireworks.jpeg"},
|
||||
{"jpg_200", "fireworks.jpeg"},
|
||||
{"pdf", "paper-100k.pdf"},
|
||||
{"html4", "html_x_4"},
|
||||
{"cp", "cp.html"},
|
||||
{"c", "fields.c"},
|
||||
{"lsp", "grammar.lsp"},
|
||||
{"xls", "kennedy.xls"},
|
||||
{"txt1", "alice29.txt"},
|
||||
{"txt2", "asyoulik.txt"},
|
||||
{"txt3", "lcet10.txt"},
|
||||
{"txt4", "plrabn12.txt"},
|
||||
{"bin", "ptt5"},
|
||||
{"sum", "sum"},
|
||||
{"man", "xargs.1"},
|
||||
{"pb", "geo.protodata"},
|
||||
{"gaviota", "kppkn.gtb"},
|
||||
}
|
||||
|
||||
// The test data files are present at this canonical URL.
|
||||
const baseURL = "https://snappy.googlecode.com/svn/trunk/testdata/"
|
||||
const baseURL = "https://raw.githubusercontent.com/google/snappy/master/testdata/"
|
||||
|
||||
func downloadTestdata(basename string) (errRet error) {
|
||||
filename := filepath.Join("testdata", basename)
|
||||
filename := filepath.Join(*testdata, basename)
|
||||
if stat, err := os.Stat(filename); err == nil && stat.Size() != 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
if !*download {
|
||||
return fmt.Errorf("test data not found; skipping benchmark without the -download flag")
|
||||
}
|
||||
// Download the official snappy C++ implementation reference test data
|
||||
// files for benchmarking.
|
||||
if err := os.Mkdir(*testdata, 0777); err != nil && !os.IsExist(err) {
|
||||
return fmt.Errorf("failed to create testdata: %s", err)
|
||||
}
|
||||
|
||||
f, err := os.Create(filename)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create %s: %s", filename, err)
|
||||
@@ -185,36 +309,27 @@ func downloadTestdata(basename string) (errRet error) {
|
||||
os.Remove(filename)
|
||||
}
|
||||
}()
|
||||
resp, err := http.Get(baseURL + basename)
|
||||
url := baseURL + basename
|
||||
resp, err := http.Get(url)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to download %s: %s", baseURL+basename, err)
|
||||
return fmt.Errorf("failed to download %s: %s", url, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if s := resp.StatusCode; s != http.StatusOK {
|
||||
return fmt.Errorf("downloading %s: HTTP status code %d (%s)", url, s, http.StatusText(s))
|
||||
}
|
||||
_, err = io.Copy(f, resp.Body)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to write %s: %s", filename, err)
|
||||
return fmt.Errorf("failed to download %s to %s: %s", url, filename, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func benchFile(b *testing.B, n int, decode bool) {
|
||||
filename := filepath.Join("testdata", testFiles[n].filename)
|
||||
if stat, err := os.Stat(filename); err != nil || stat.Size() == 0 {
|
||||
if !*download {
|
||||
b.Fatal("test data not found; skipping benchmark without the -download flag")
|
||||
}
|
||||
// Download the official snappy C++ implementation reference test data
|
||||
// files for benchmarking.
|
||||
if err := os.Mkdir("testdata", 0777); err != nil && !os.IsExist(err) {
|
||||
b.Fatalf("failed to create testdata: %s", err)
|
||||
}
|
||||
for _, tf := range testFiles {
|
||||
if err := downloadTestdata(tf.filename); err != nil {
|
||||
b.Fatalf("failed to download testdata: %s", err)
|
||||
}
|
||||
}
|
||||
if err := downloadTestdata(testFiles[n].filename); err != nil {
|
||||
b.Fatalf("failed to download testdata: %s", err)
|
||||
}
|
||||
data := readFile(b, filename)
|
||||
data := readFile(b, filepath.Join(*testdata, testFiles[n].filename))
|
||||
if decode {
|
||||
benchDecode(b, data)
|
||||
} else {
|
||||
@@ -235,12 +350,6 @@ func Benchmark_UFlat8(b *testing.B) { benchFile(b, 8, true) }
|
||||
func Benchmark_UFlat9(b *testing.B) { benchFile(b, 9, true) }
|
||||
func Benchmark_UFlat10(b *testing.B) { benchFile(b, 10, true) }
|
||||
func Benchmark_UFlat11(b *testing.B) { benchFile(b, 11, true) }
|
||||
func Benchmark_UFlat12(b *testing.B) { benchFile(b, 12, true) }
|
||||
func Benchmark_UFlat13(b *testing.B) { benchFile(b, 13, true) }
|
||||
func Benchmark_UFlat14(b *testing.B) { benchFile(b, 14, true) }
|
||||
func Benchmark_UFlat15(b *testing.B) { benchFile(b, 15, true) }
|
||||
func Benchmark_UFlat16(b *testing.B) { benchFile(b, 16, true) }
|
||||
func Benchmark_UFlat17(b *testing.B) { benchFile(b, 17, true) }
|
||||
func Benchmark_ZFlat0(b *testing.B) { benchFile(b, 0, false) }
|
||||
func Benchmark_ZFlat1(b *testing.B) { benchFile(b, 1, false) }
|
||||
func Benchmark_ZFlat2(b *testing.B) { benchFile(b, 2, false) }
|
||||
@@ -253,9 +362,3 @@ func Benchmark_ZFlat8(b *testing.B) { benchFile(b, 8, false) }
|
||||
func Benchmark_ZFlat9(b *testing.B) { benchFile(b, 9, false) }
|
||||
func Benchmark_ZFlat10(b *testing.B) { benchFile(b, 10, false) }
|
||||
func Benchmark_ZFlat11(b *testing.B) { benchFile(b, 11, false) }
|
||||
func Benchmark_ZFlat12(b *testing.B) { benchFile(b, 12, false) }
|
||||
func Benchmark_ZFlat13(b *testing.B) { benchFile(b, 13, false) }
|
||||
func Benchmark_ZFlat14(b *testing.B) { benchFile(b, 14, false) }
|
||||
func Benchmark_ZFlat15(b *testing.B) { benchFile(b, 15, false) }
|
||||
func Benchmark_ZFlat16(b *testing.B) { benchFile(b, 16, false) }
|
||||
func Benchmark_ZFlat17(b *testing.B) { benchFile(b, 17, false) }
|
||||
|
||||
Reference in New Issue
Block a user