v1

package
v0.6.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Feb 18, 2021 License: Apache-2.0 Imports: 14 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

View Source 
var (
	// Gzip is the gnu zip compression pool
	Gzip = GzipPool{/* contains filtered or unexported fields */}
	// Lz4_64k is the l4z compression pool, with 64k buffer size
	Lz4_64k = LZ4Pool{/* contains filtered or unexported fields */}
	// Lz4_256k uses 256k buffer
	Lz4_256k = LZ4Pool{/* contains filtered or unexported fields */}
	// Lz4_1M uses 1M buffer
	Lz4_1M = LZ4Pool{/* contains filtered or unexported fields */}
	// Lz4_4M uses 4M buffer
	Lz4_4M = LZ4Pool{/* contains filtered or unexported fields */}
	// Snappy is the snappy compression pool
	Snappy SnappyPool
	// Noop is the no compression pool
	Noop NoopPool
	// Zstd Pool
	Zstd = ZstdPool{}

	// BytesBufferPool is a bytes buffer used for lines decompressed.
	// Buckets [0.5KB,1KB,2KB,4KB,8KB]
	BytesBufferPool = pool.New(1<<9, 1<<13, 2, func(size int) interface{} { return make([]byte, 0, size) })
)

Functions

func AppendBlockData 

func AppendBlockData(ctx context.Context, w backend.Writer, meta *backend.BlockMeta, tracker backend.AppendTracker, buffer []byte) (backend.AppendTracker, error)

AppendBlockData appends the bytes passed to the block data

func BloomName 

func BloomName(shard int) string

BloomName returns v0 name

func NameIndex 

func NameIndex() string

NameIndex returns v0 name

func NameObjects 

func NameObjects() string

NameObjects returns v0 name

func NewBufferedAppender 

func NewBufferedAppender(writer io.Writer, encoding backend.Encoding, indexDownsample int, totalObjectsEstimate int) (common.Appender, error)

NewBufferedAppender returns an bufferedAppender. This appender builds a writes to 

the provided writer and also builds a downsampled records slice.

func NewIndexReader 

func NewIndexReader(index []byte) (common.IndexReader, error)

NewIndexReader returns an index reader for a byte slice of marshalled ordered records. The index has not changed between v0 and v1.

func NewPageReader 

func NewPageReader(r io.ReaderAt, encoding backend.Encoding) (common.PageReader, error)

NewPageReader constructs a v1 PageReader that handles compression

func NewPagedFinder 

func NewPagedFinder(index common.IndexReader, r common.PageReader, combiner common.ObjectCombiner) common.Finder

NewPagedFinder returns a v0.pagedFinder. There are no changes to logic from the v0 finder and all compression changes are handled in the pageReader

func NewPagedIterator 

func NewPagedIterator(chunkSizeBytes uint32, indexReader common.IndexReader, pageReader common.PageReader) common.Iterator

NewPagedIterator returns a v0.backendIterator. There are no changes to logic from the v0 iterator and all compression changes are handled in the pageReader

func WriteBlockData 

func WriteBlockData(ctx context.Context, w backend.Writer, meta *backend.BlockMeta, r io.Reader, size int64) error

WriteBlockData writes the data object from an io.Reader to the backend.Writer

func WriteBlockMeta 

func WriteBlockMeta(ctx context.Context, w backend.Writer, meta *backend.BlockMeta, records []*common.Record, b *common.ShardedBloomFilter) error

WriteBlockMeta writes the bloom filter, meta and index to the passed in backend.Writer

Types

type GzipPool 

type GzipPool struct {
	// contains filtered or unexported fields
}

GzipPool is a gun zip compression pool

func (*GzipPool) Encoding 

func (pool *GzipPool) Encoding() backend.Encoding

Encoding implements WriterPool and ReaderPool

func (*GzipPool) GetReader 

func (pool *GzipPool) GetReader(src io.Reader) (io.Reader, error)

GetReader gets or creates a new CompressionReader and reset it to read from src

func (*GzipPool) GetWriter 

func (pool *GzipPool) GetWriter(dst io.Writer) (io.WriteCloser, error)

GetWriter gets or creates a new CompressionWriter and reset it to write to dst

func (*GzipPool) PutReader 

func (pool *GzipPool) PutReader(reader io.Reader)

PutReader places back in the pool a CompressionReader

func (*GzipPool) PutWriter 

func (pool *GzipPool) PutWriter(writer io.WriteCloser)

PutWriter places back in the pool a CompressionWriter

func (*GzipPool) ResetReader 

func (pool *GzipPool) ResetReader(src io.Reader, resetReader io.Reader) (io.Reader, error)

ResetReader implements ReaderPool

func (*GzipPool) ResetWriter 

func (pool *GzipPool) ResetWriter(dst io.Writer, resetWriter io.WriteCloser) (io.WriteCloser, error)

ResetWriter implements WriterPool

type LZ4Pool 

type LZ4Pool struct {
	// contains filtered or unexported fields
}

LZ4Pool is an pool...of lz4s...

func (*LZ4Pool) Encoding 

func (pool *LZ4Pool) Encoding() backend.Encoding

Encoding implements WriterPool and ReaderPool

func (*LZ4Pool) GetReader 

func (pool *LZ4Pool) GetReader(src io.Reader) (io.Reader, error)

GetReader gets or creates a new CompressionReader and reset it to read from src

func (*LZ4Pool) GetWriter 

func (pool *LZ4Pool) GetWriter(dst io.Writer) (io.WriteCloser, error)

GetWriter gets or creates a new CompressionWriter and reset it to write to dst

func (*LZ4Pool) PutReader 

func (pool *LZ4Pool) PutReader(reader io.Reader)

PutReader places back in the pool a CompressionReader

func (*LZ4Pool) PutWriter 

func (pool *LZ4Pool) PutWriter(writer io.WriteCloser)

PutWriter places back in the pool a CompressionWriter

func (*LZ4Pool) ResetReader 

func (pool *LZ4Pool) ResetReader(src io.Reader, resetReader io.Reader) (io.Reader, error)

ResetReader implements ReaderPool

func (*LZ4Pool) ResetWriter 

func (pool *LZ4Pool) ResetWriter(dst io.Writer, resetWriter io.WriteCloser) (io.WriteCloser, error)

ResetWriter implements WriterPool

type NoopPool 

type NoopPool struct{}

NoopPool is for people who think compression is for the weak

func (*NoopPool) Encoding 

func (pool *NoopPool) Encoding() backend.Encoding

Encoding implements WriterPool and ReaderPool

func (*NoopPool) GetReader 

func (pool *NoopPool) GetReader(src io.Reader) (io.Reader, error)

GetReader gets or creates a new CompressionReader and reset it to read from src

func (*NoopPool) GetWriter 

func (pool *NoopPool) GetWriter(dst io.Writer) (io.WriteCloser, error)

GetWriter gets or creates a new CompressionWriter and reset it to write to dst

func (*NoopPool) PutReader 

func (pool *NoopPool) PutReader(reader io.Reader)

PutReader places back in the pool a CompressionReader

func (*NoopPool) PutWriter 

func (pool *NoopPool) PutWriter(writer io.WriteCloser)

PutWriter places back in the pool a CompressionWriter

func (*NoopPool) ResetReader 

func (pool *NoopPool) ResetReader(src io.Reader, resetReader io.Reader) (io.Reader, error)

ResetReader implements ReaderPool

func (*NoopPool) ResetWriter 

func (pool *NoopPool) ResetWriter(dst io.Writer, resetWriter io.WriteCloser) (io.WriteCloser, error)

ResetWriter implements WriterPool

type ReaderPool 

type ReaderPool interface {
	GetReader(io.Reader) (io.Reader, error)
	PutReader(io.Reader)
	ResetReader(src io.Reader, resetReader io.Reader) (io.Reader, error)
	Encoding() backend.Encoding
}

ReaderPool similar to WriterPool but for reading chunks.

type SnappyPool 

type SnappyPool struct {
	// contains filtered or unexported fields
}

SnappyPool is a really cool looking pool. Dang that pool is _snappy_.

func (*SnappyPool) Encoding 

func (pool *SnappyPool) Encoding() backend.Encoding

Encoding implements WriterPool and ReaderPool

func (*SnappyPool) GetReader 

func (pool *SnappyPool) GetReader(src io.Reader) (io.Reader, error)

GetReader gets or creates a new CompressionReader and reset it to read from src

func (*SnappyPool) GetWriter 

func (pool *SnappyPool) GetWriter(dst io.Writer) (io.WriteCloser, error)

GetWriter gets or creates a new CompressionWriter and reset it to write to dst

func (*SnappyPool) PutReader 

func (pool *SnappyPool) PutReader(reader io.Reader)

PutReader places back in the pool a CompressionReader

func (*SnappyPool) PutWriter 

func (pool *SnappyPool) PutWriter(writer io.WriteCloser)

PutWriter places back in the pool a CompressionWriter

func (*SnappyPool) ResetReader 

func (pool *SnappyPool) ResetReader(src io.Reader, resetReader io.Reader) (io.Reader, error)

ResetReader implements ReaderPool

func (*SnappyPool) ResetWriter 

func (pool *SnappyPool) ResetWriter(dst io.Writer, resetWriter io.WriteCloser) (io.WriteCloser, error)

ResetWriter implements WriterPool

type WriterPool 

type WriterPool interface {
	GetWriter(io.Writer) (io.WriteCloser, error)
	PutWriter(io.WriteCloser)
	ResetWriter(dst io.Writer, resetWriter io.WriteCloser) (io.WriteCloser, error)
	Encoding() backend.Encoding
}

WriterPool is a pool of io.Writer This is used by every chunk to avoid unnecessary allocations.

type ZstdPool 

type ZstdPool struct {
}

ZstdPool is a zstd compression pool

func (*ZstdPool) Encoding 

func (pool *ZstdPool) Encoding() backend.Encoding

Encoding implements WriterPool and ReaderPool

func (*ZstdPool) GetReader 

func (pool *ZstdPool) GetReader(src io.Reader) (io.Reader, error)

GetReader gets or creates a new CompressionReader and reset it to read from src

func (*ZstdPool) GetWriter 

func (pool *ZstdPool) GetWriter(dst io.Writer) (io.WriteCloser, error)

GetWriter gets or creates a new CompressionWriter and reset it to write to dst

func (*ZstdPool) PutReader 

func (pool *ZstdPool) PutReader(reader io.Reader)

PutReader places back in the pool a CompressionReader

func (*ZstdPool) PutWriter 

func (pool *ZstdPool) PutWriter(writer io.WriteCloser)

PutWriter places back in the pool a CompressionWriter

func (*ZstdPool) ResetReader 

func (pool *ZstdPool) ResetReader(src io.Reader, resetReader io.Reader) (io.Reader, error)

ResetReader implements ReaderPool

func (*ZstdPool) ResetWriter 

func (pool *ZstdPool) ResetWriter(dst io.Writer, resetWriter io.WriteCloser) (io.WriteCloser, error)

ResetWriter implements WriterPool

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.