Versions in this module Expand all Collapse all v0 v0.40.4 May 19, 2022 Changes in this version + const AWSStorageVersion + const StorageVersion + var ErrAlreadyFinished = errors.New("already Finished") + var ErrBufferFull = errors.New("buffer full") + var ErrChunkAlreadyWritten = errors.New("chunk already written") + var ErrConcurrentManifestWriteDuringOverwrite = errors.New("concurrent manifest write during manifest overwrite") + var ErrCorruptManifest = errors.New("corrupt manifest") + var ErrFetchFailure = errors.New("fetch failed") + var ErrInvalidTableFile = errors.New("invalid or corrupt table file") + var ErrNoChunkSource = errors.New("no chunk source") + var ErrNoReader = errors.New("could not get reader") + var ErrNotEnoughBytes = errors.New("reader did not return enough bytes") + var ErrNotFinished = errors.New("not finished") + var ErrSpecWithoutChunkSource = errors.New("manifest referenced table file for which there is no chunkSource.") + var ErrUnreadableManifest = errors.New("could not read file manifest") + var ErrUnsupportedManifestAppendixOption = errors.New("unsupported manifest appendix option") + var ErrWrongBufferSize = errors.New("buffer length and/or capacity incorrect for chunkCount specified in footer") + var ErrWrongCopySize = errors.New("could not copy enough bytes") + var GlobalMmapStats = &mmapStats + func GetTableIndexPrefixes(rd io.ReadSeeker) (prefixes []uint64, err error) + func GuessPrefixOrdinal(prefix uint64, n uint32) int + func IterChunks(rd io.ReadSeeker, cb func(chunk chunks.Chunk) (stop bool, err error)) error + func MaybeMigrateFileManifest(ctx context.Context, dir string) (bool, error) + func NewIndexTransformer(src io.Reader, chunkCount int) io.Reader + func OverwriteStoreManifest(ctx context.Context, store *NomsBlockStore, root hash.Hash, ...) (err error) + func ReadTableFooter(rd io.ReadSeeker) (chunkCount uint32, totalUncompressedData uint64, err error) + func ReadTableIndexByCopy(rd io.ReadSeeker, q MemoryQuotaProvider) (onHeapTableIndex, error) + func ValidateAddr(s string) bool + func WriteChunks(chunks []chunks.Chunk) (string, []byte, error) + type BlockBufferByteSink struct + func NewBlockBufferTableSink(blockSize int) *BlockBufferByteSink + func (sink *BlockBufferByteSink) Flush(wr io.Writer) (err error) + func (sink *BlockBufferByteSink) FlushToFile(path string) (err error) + func (sink *BlockBufferByteSink) Write(src []byte) (int, error) + type BufferedFileByteSink struct + func NewBufferedFileByteSink(tempDir string, blockSize, chBufferSize int) (*BufferedFileByteSink, error) + func (sink *BufferedFileByteSink) Flush(wr io.Writer) (err error) + func (sink *BufferedFileByteSink) FlushToFile(path string) (err error) + func (sink *BufferedFileByteSink) Write(src []byte) (int, error) + type ByteSink interface + Flush func(wr io.Writer) error + FlushToFile func(path string) error + type CmpChunkTableWriter struct + func NewCmpChunkTableWriter(tempDir string) (*CmpChunkTableWriter, error) + func (tw *CmpChunkTableWriter) AddCmpChunk(c CompressedChunk) error + func (tw *CmpChunkTableWriter) ChunkCount() uint32 + func (tw *CmpChunkTableWriter) ContentLength() uint64 + func (tw *CmpChunkTableWriter) Finish() (string, error) + func (tw *CmpChunkTableWriter) Flush(wr io.Writer) error + func (tw *CmpChunkTableWriter) FlushToFile(path string) error + func (tw *CmpChunkTableWriter) GetMD5() []byte + func (tw *CmpChunkTableWriter) Size() int + type CompressedChunk struct + CompressedData []byte + FullCompressedChunk []byte + H hash.Hash + var EmptyCompressedChunk CompressedChunk + func ChunkToCompressedChunk(chunk chunks.Chunk) CompressedChunk + func NewCompressedChunk(h hash.Hash, buff []byte) (CompressedChunk, error) + func (cmp CompressedChunk) Hash() hash.Hash + func (cmp CompressedChunk) IsEmpty() bool + func (cmp CompressedChunk) ToChunk() (chunks.Chunk, error) + type FixedBufferByteSink struct + func NewFixedBufferTableSink(buff []byte) *FixedBufferByteSink + func (sink *FixedBufferByteSink) Flush(wr io.Writer) error + func (sink *FixedBufferByteSink) FlushToFile(path string) (err error) + func (sink *FixedBufferByteSink) Write(src []byte) (int, error) + type GenerationalNBS struct + func NewGenerationalCS(oldGen, newGen *NomsBlockStore) *GenerationalNBS + func (gcs *GenerationalNBS) AddTableFilesToManifest(ctx context.Context, fileIdToNumChunks map[string]int) error + func (gcs *GenerationalNBS) Close() error + func (gcs *GenerationalNBS) Commit(ctx context.Context, current, last hash.Hash) (bool, error) + func (gcs *GenerationalNBS) Get(ctx context.Context, h hash.Hash) (chunks.Chunk, error) + func (gcs *GenerationalNBS) GetMany(ctx context.Context, hashes hash.HashSet, ...) error + func (gcs *GenerationalNBS) GetManyCompressed(ctx context.Context, hashes hash.HashSet, ...) error + func (gcs *GenerationalNBS) Has(ctx context.Context, h hash.Hash) (bool, error) + func (gcs *GenerationalNBS) HasMany(ctx context.Context, hashes hash.HashSet) (absent hash.HashSet, err error) + func (gcs *GenerationalNBS) NewGen() chunks.ChunkStoreGarbageCollector + func (gcs *GenerationalNBS) OldGen() chunks.ChunkStoreGarbageCollector + func (gcs *GenerationalNBS) PruneTableFiles(ctx context.Context) error + func (gcs *GenerationalNBS) Put(ctx context.Context, c chunks.Chunk) error + func (gcs *GenerationalNBS) Rebase(ctx context.Context) error + func (gcs *GenerationalNBS) Root(ctx context.Context) (hash.Hash, error) + func (gcs *GenerationalNBS) SetRootChunk(ctx context.Context, root, previous hash.Hash) error + func (gcs *GenerationalNBS) Size(ctx context.Context) (uint64, error) + func (gcs *GenerationalNBS) Sources(ctx context.Context) (hash.Hash, []TableFile, []TableFile, error) + func (gcs *GenerationalNBS) Stats() interface{} + func (gcs *GenerationalNBS) StatsSummary() string + func (gcs *GenerationalNBS) SupportedOperations() TableFileStoreOps + func (gcs *GenerationalNBS) Version() string + func (gcs *GenerationalNBS) WriteTableFile(ctx context.Context, fileId string, numChunks int, contentHash []byte, ...) error + type HashingByteSink struct + func NewHashingByteSink(backingSink ByteSink) *HashingByteSink + func (sink *HashingByteSink) Flush(wr io.Writer) error + func (sink *HashingByteSink) FlushToFile(path string) error + func (sink *HashingByteSink) GetMD5() []byte + func (sink *HashingByteSink) Size() uint64 + func (sink *HashingByteSink) Write(src []byte) (int, error) + type ManifestAppendixOption int + const ManifestAppendixOption_Append + const ManifestAppendixOption_Set + const ManifestAppendixOption_Unspecified + type ManifestInfo interface + GetAppendixTableSpecInfo func(i int) TableSpecInfo + GetGCGen func() string + GetLock func() string + GetRoot func() hash.Hash + GetTableSpecInfo func(i int) TableSpecInfo + GetVersion func() string + NumAppendixSpecs func() int + NumTableSpecs func() int + func ParseManifest(r io.Reader) (ManifestInfo, error) + type MemoryQuotaProvider interface + AcquireQuota func(ctx context.Context, memory uint64) error + ReleaseQuota func(memory uint64) error + Usage func() uint64 + type NBSCompressedChunkStore interface + GetManyCompressed func(context.Context, hash.HashSet, func(context.Context, CompressedChunk)) error + type NBSMetricWrapper struct + func NewNBSMetricWrapper(nbs *NomsBlockStore) *NBSMetricWrapper + func (nbsMW *NBSMetricWrapper) AddTableFilesToManifest(ctx context.Context, fileIdToNumChunks map[string]int) error + func (nbsMW *NBSMetricWrapper) GetManyCompressed(ctx context.Context, hashes hash.HashSet, ...) error + func (nbsMW *NBSMetricWrapper) MarkAndSweepChunks(ctx context.Context, last hash.Hash, keepChunks <-chan []hash.Hash, ...) error + func (nbsMW *NBSMetricWrapper) PruneTableFiles(ctx context.Context) error + func (nbsMW *NBSMetricWrapper) SetRootChunk(ctx context.Context, root, previous hash.Hash) error + func (nbsMW *NBSMetricWrapper) Size(ctx context.Context) (uint64, error) + func (nbsMW *NBSMetricWrapper) Sources(ctx context.Context) (hash.Hash, []TableFile, []TableFile, error) + func (nbsMW *NBSMetricWrapper) SupportedOperations() TableFileStoreOps + func (nbsMW *NBSMetricWrapper) WriteTableFile(ctx context.Context, fileId string, numChunks int, contentHash []byte, ...) error + type NomsBlockStore struct + func NewAWSStore(ctx context.Context, nbfVerStr string, table, ns, bucket string, s3 s3svc, ...) (*NomsBlockStore, error) + func NewAWSStoreWithMMapIndex(ctx context.Context, nbfVerStr string, table, ns, bucket string, s3 s3svc, ...) (*NomsBlockStore, error) + func NewBSStore(ctx context.Context, nbfVerStr string, bs blobstore.Blobstore, ...) (*NomsBlockStore, error) + func NewGCSStore(ctx context.Context, nbfVerStr string, bucketName, path string, ...) (*NomsBlockStore, error) + func NewLocalStore(ctx context.Context, nbfVerStr string, dir string, memTableSize uint64, ...) (*NomsBlockStore, error) + func (nbs *NomsBlockStore) AddTableFilesToManifest(ctx context.Context, fileIdToNumChunks map[string]int) error + func (nbs *NomsBlockStore) CalcReads(hashes hash.HashSet, blockSize uint64) (reads int, split bool, err error) + func (nbs *NomsBlockStore) Close() error + func (nbs *NomsBlockStore) Commit(ctx context.Context, current, last hash.Hash) (success bool, err error) + func (nbs *NomsBlockStore) Count() (uint32, error) + func (nbs *NomsBlockStore) Get(ctx context.Context, h hash.Hash) (chunks.Chunk, error) + func (nbs *NomsBlockStore) GetChunkLocations(hashes hash.HashSet) (map[hash.Hash]map[hash.Hash]Range, error) + func (nbs *NomsBlockStore) GetMany(ctx context.Context, hashes hash.HashSet, ...) error + func (nbs *NomsBlockStore) GetManyCompressed(ctx context.Context, hashes hash.HashSet, ...) error + func (nbs *NomsBlockStore) Has(ctx context.Context, h hash.Hash) (bool, error) + func (nbs *NomsBlockStore) HasMany(ctx context.Context, hashes hash.HashSet) (hash.HashSet, error) + func (nbs *NomsBlockStore) MarkAndSweepChunks(ctx context.Context, last hash.Hash, keepChunks <-chan []hash.Hash, ...) error + func (nbs *NomsBlockStore) PruneTableFiles(ctx context.Context) (err error) + func (nbs *NomsBlockStore) Put(ctx context.Context, c chunks.Chunk) error + func (nbs *NomsBlockStore) Rebase(ctx context.Context) error + func (nbs *NomsBlockStore) Root(ctx context.Context) (hash.Hash, error) + func (nbs *NomsBlockStore) SetRootChunk(ctx context.Context, root, previous hash.Hash) error + func (nbs *NomsBlockStore) Size(ctx context.Context) (uint64, error) + func (nbs *NomsBlockStore) Sources(ctx context.Context) (hash.Hash, []TableFile, []TableFile, error) + func (nbs *NomsBlockStore) Stats() interface{} + func (nbs *NomsBlockStore) StatsSummary() string + func (nbs *NomsBlockStore) SupportedOperations() TableFileStoreOps + func (nbs *NomsBlockStore) UpdateManifest(ctx context.Context, updates map[hash.Hash]uint32) (mi ManifestInfo, err error) + func (nbs *NomsBlockStore) UpdateManifestWithAppendix(ctx context.Context, updates map[hash.Hash]uint32, ...) (mi ManifestInfo, err error) + func (nbs *NomsBlockStore) Version() string + func (nbs *NomsBlockStore) WithoutConjoiner() *NomsBlockStore + func (nbs *NomsBlockStore) WriteTableFile(ctx context.Context, fileId string, numChunks int, contentHash []byte, ...) error + type OffsetsReader struct + func NewOffsetsReader(lengthsReader io.Reader) *OffsetsReader + func (tra *OffsetsReader) Read(p []byte) (n int, err error) + type Range struct + Length uint32 + Offset uint64 + type Stats struct + AddressesPerHas metrics.Histogram + BytesPerConjoin metrics.Histogram + BytesPerPersist metrics.Histogram + ChunksPerConjoin metrics.Histogram + ChunksPerGet metrics.Histogram + ChunksPerPersist metrics.Histogram + CommitLatency metrics.Histogram + CompressedChunkBytesPerPersist metrics.Histogram + ConjoinLatency metrics.Histogram + DynamoBytesPerRead metrics.Histogram + DynamoReadLatency metrics.Histogram + FileBytesPerRead metrics.Histogram + FileReadLatency metrics.Histogram + GetLatency metrics.Histogram + HasLatency metrics.Histogram + IndexBytesPerRead metrics.Histogram + IndexReadLatency metrics.Histogram + MemBytesPerRead metrics.Histogram + MemReadLatency metrics.Histogram + OpenLatency metrics.Histogram + PersistLatency metrics.Histogram + PutLatency metrics.Histogram + ReadManifestLatency metrics.Histogram + S3BytesPerRead metrics.Histogram + S3ReadLatency metrics.Histogram + TablesPerConjoin metrics.Histogram + UncompressedChunkBytesPerPersist metrics.Histogram + WriteManifestLatency metrics.Histogram + func NewStats() *Stats + func (s *Stats) Clone() Stats + func (s Stats) String() string + type TableFile interface + FileID func() string + NumChunks func() int + Open func(ctx context.Context) (io.ReadCloser, uint64, error) + type TableFileStore interface + AddTableFilesToManifest func(ctx context.Context, fileIdToNumChunks map[string]int) error + PruneTableFiles func(ctx context.Context) error + SetRootChunk func(ctx context.Context, root, previous hash.Hash) error + Size func(ctx context.Context) (uint64, error) + Sources func(ctx context.Context) (hash.Hash, []TableFile, []TableFile, error) + SupportedOperations func() TableFileStoreOps + WriteTableFile func(ctx context.Context, fileId string, numChunks int, contentHash []byte, ...) error + type TableFileStoreOps struct + CanGC bool + CanPrune bool + CanRead bool + CanWrite bool + type TableSpecInfo interface + GetChunkCount func() uint32 + GetName func() string + type UnlimitedQuotaProvider struct + func NewUnlimitedMemQuotaProvider() *UnlimitedQuotaProvider + func (q *UnlimitedQuotaProvider) AcquireQuota(ctx context.Context, memory uint64) error + func (q *UnlimitedQuotaProvider) ReleaseQuota(memory uint64) error + func (q *UnlimitedQuotaProvider) Usage() uint64
Go to main page