carlog

package
v0.0.0-...-9f36c07 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jun 30, 2024 License: Apache-2.0, MIT Imports: 34 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	HeadName = "head"
	HeadSize = 512

	LevelIndex     = "index.level"
	BsstIndex      = "index.bsst"
	BsstIndexCanon = "fil.bsst"
	HashSample     = "sample.mhlist"

	BlockLog = "blklog.car"
	FilCar   = "fil.car"
)
View Source 
const MaxEntryLen = 1 << (64 - 40)

Variables

View Source 
var ErrAlreadyOffloaded = xerrors.Errorf("group already offloaded")
View Source 
var ErrReadOnly = errors.New("already read-only")

Functions

func LoadMHList 

func LoadMHList(filePath string) ([]multihash.Multihash, error)

func SaveMHList 

func SaveMHList(filePath string, list []multihash.Multihash) error

Types

type BSSTIndex 

type BSSTIndex struct {
	// contains filtered or unexported fields
}

func CreateBSSTIndex 

func CreateBSSTIndex(path string, index IndexSource) (*BSSTIndex, error)

func OpenBSSTIndex 

func OpenBSSTIndex(path string) (*BSSTIndex, error)

func (*BSSTIndex) Close 

func (b *BSSTIndex) Close() error

func (*BSSTIndex) Entries 

func (b *BSSTIndex) Entries() (int64, error)

func (*BSSTIndex) Get 

func (b *BSSTIndex) Get(c []mh.Multihash) ([]int64, error)

func (*BSSTIndex) Has 

func (b *BSSTIndex) Has(c []mh.Multihash) ([]bool, error)

func (*BSSTIndex) List 

func (b *BSSTIndex) List(f func(c mh.Multihash, offs []int64) error) error

type CarLog 

type CarLog struct {

	// index = dir, data = file
	IndexPath, DataPath string
	// contains filtered or unexported fields
}

CarLog is a .car file which is storing a flat layer of blocks, with a wide top tree and a single root * Before the CarLog is finalized, the root CID is a placeholher Identity hash * Blocks are stored layer-by-layer, from the bottom, left to right * Transforming into depth-first .car is very cheap with the head file * NOT THREAD SAFE FOR WRITING!!

  • Reads can happen in parallel with writing

* One tx at a time * Not considered written until committed

func Create 

func Create(staging CarStorageProvider, indexPath, dataPath string, _ TruncCleanup) (*CarLog, error)

func Open 

func Open(staging CarStorageProvider, indexPath, dataPath string, tc TruncCleanup) (*CarLog, error)

func (*CarLog) Close 

func (j *CarLog) Close() error

func (*CarLog) Commit 

func (j *CarLog) Commit() (int64, error)

func (*CarLog) Delete 

func (j *CarLog) Delete(c []mh.Multihash) error

func (*CarLog) FinDataReload 

func (j *CarLog) FinDataReload(ctx context.Context, blkEnts int64, carSz int64) error

func (*CarLog) Finalize 

func (j *CarLog) Finalize(ctx context.Context) error

func (*CarLog) HashSample 

func (j *CarLog) HashSample() ([]mh.Multihash, error)

func (*CarLog) LoadData 

func (j *CarLog) LoadData(ctx context.Context, car io.Reader, sz int64) error

func (*CarLog) MarkReadOnly 

func (j *CarLog) MarkReadOnly() error

func (*CarLog) Offload 

func (j *CarLog) Offload() error

func (*CarLog) OffloadData 

func (j *CarLog) OffloadData() error

func (*CarLog) Put 

func (j *CarLog) Put(c []mh.Multihash, b []blocks.Block) error

func (*CarLog) View 

func (j *CarLog) View(c []mh.Multihash, cb func(cidx int, found bool, data []byte) error) error

func (*CarLog) WriteCar 

func (j *CarLog) WriteCar(w io.Writer) (int64, cid.Cid, error)

returns car size and root cid

type CarStorageProvider 

type CarStorageProvider interface {
	Upload(ctx context.Context, size int64, src func(writer io.Writer) error) error
	Has(ctx context.Context) (bool, error)

	io.ReaderAt
}
type Head struct {
	Version int64

	// something that's not zero
	Valid bool

	// byte offset just after the last retired op. If finalized, but layers aren't set
	// this points to the end of first (bottom) layer
	RetiredAt int64

	DataStart int64

	// byte offset of the start of the second layer
	DataEnd int64

	ReadOnly  bool // if true, no more writes are allowed
	Finalized bool // if true, no more writes are allowed, and the bsst index is finalized
	Offloaded bool // if true, the data file is offloaded to external storage, only hash samples are kept
	External  bool // if true, the data is moved to external storage on finalize

	// Layer stats, set after Finalized (Finalized can be true and layers may still not be set)
	LayerOffsets []int64 // byte offsets of the start of each layer
}

Head is the on-disk head object. CBOR-map-serialized. Must fit in 

HeadSize bytes. Null-Padded to exactly HeadSize

func (*Head) MarshalCBOR 

func (t *Head) MarshalCBOR(w io.Writer) error

func (*Head) UnmarshalCBOR 

func (t *Head) UnmarshalCBOR(r io.Reader) (err error)

type IndexSource 

type IndexSource interface {
	bsst.Source
	Entries() (int64, error)
}

type LevelDBIndex 

type LevelDBIndex struct {
	*leveldb.DB
}

func OpenLevelDBIndex 

func OpenLevelDBIndex(path string, create bool) (*LevelDBIndex, error)

func (*LevelDBIndex) Close 

func (l *LevelDBIndex) Close() error

func (*LevelDBIndex) Del 

func (l *LevelDBIndex) Del(c []multihash.Multihash) error

func (*LevelDBIndex) Entries 

func (l *LevelDBIndex) Entries() (int64, error)

func (*LevelDBIndex) Get 

func (l *LevelDBIndex) Get(c []multihash.Multihash) ([]int64, error)

Get returns offsets to data, -1 if not found

func (*LevelDBIndex) Has 

func (l *LevelDBIndex) Has(c []multihash.Multihash) ([]bool, error)

func (*LevelDBIndex) List 

func (l *LevelDBIndex) List(f func(c multihash.Multihash, offs []int64) error) error

func (*LevelDBIndex) Put 

func (l *LevelDBIndex) Put(c []multihash.Multihash, offs []int64) error

func (*LevelDBIndex) ToTruncate 

func (l *LevelDBIndex) ToTruncate(atOrAbove int64) ([]multihash.Multihash, error)

type ReadableIndex 

type ReadableIndex interface {
	// todo maybe callback calling with sequential indexes of what we don't have
	//  to pipeline better?
	Has(c []mh.Multihash) ([]bool, error)

	// Get returns offsets to data, -1 if not found
	Get(c []mh.Multihash) ([]int64, error)

	// bsst creation
	Entries() (int64, error)
	bsst.Source

	Close() error
}

type TruncCleanup 

type TruncCleanup func(to int64, h []mh.Multihash) error

type WritableIndex 

type WritableIndex interface {
	// Put records entries in the index
	// sync for now, todo
	// -1 offset means 'skip'
	Put(c []mh.Multihash, offs []int64) error

	Del(c []mh.Multihash) error

	// Truncate returns a list of multihashes to remove from the index
	ToTruncate(atOrAbove int64) ([]mh.Multihash, error)

	Close() error
}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.