odb

package
v0.16.2 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Dec 23, 2024 License: Apache-2.0 Imports: 28 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	LargeSize = 5 << 20 // 5M
)
View Source 
const (
	MiByte int64 = 1048576
)

Variables

View Source 
var (
	ErrUncacheableObject = errors.New("uncacheable object")
)

Functions

func Copy 

func Copy(dst io.Writer, src io.Reader) (written int64, err error)

Copy copy reader to writer

func NewPackedUnpacker 

func NewPackedUnpacker(root string) (*packedUnpacker, error)

root: /home/zeta/repositories/001/10001.zeta/incoming/quarantine-1111/metadata

func OssRemoveFiles 

func OssRemoveFiles(ctx context.Context, b oss.Bucket, rid int64) error

Types

type CacheDB 

type CacheDB interface {
	Object(ctx context.Context, rid int64, oid plumbing.Hash) (any, error)
	Commit(ctx context.Context, rid int64, oid plumbing.Hash) (*object.Commit, error)
	Tree(ctx context.Context, rid int64, oid plumbing.Hash) (*object.Tree, error)
	Fragments(ctx context.Context, rid int64, oid plumbing.Hash) (*object.Fragments, error)
	Tag(ctx context.Context, rid int64, oid plumbing.Hash) (*object.Tag, error)
	Store(ctx context.Context, rid int64, a any) error
	Mark(rid int64, oid plumbing.Hash)
	Exist(rid int64, oid plumbing.Hash) bool
}

func NewCacheDB 

func NewCacheDB(numCounters int64, maxCost int64, bufferItems int64) (CacheDB, error)

type DB 

type DB interface {
	Commit(ctx context.Context, oid plumbing.Hash) (cc *object.Commit, err error)
	Tree(ctx context.Context, oid plumbing.Hash) (t *object.Tree, err error)
	Fragments(ctx context.Context, oid plumbing.Hash) (*object.Fragments, error)
	Tag(ctx context.Context, oid plumbing.Hash) (*object.Tag, error)
	Objects(ctx context.Context, oid plumbing.Hash) (a any, err error)
	Open(ctx context.Context, oid plumbing.Hash, start int64) (sr backend.SizeReader, err error)
	Blob(ctx context.Context, oid plumbing.Hash) (b *object.Blob, err error)
	Push(ctx context.Context, oid plumbing.Hash) error // Push object to OSS
	WriteDirect(ctx context.Context, oid plumbing.Hash, r io.Reader, size int64) (int64, error)
	Stat(ctx context.Context, oid plumbing.Hash) (*oss.Stat, error)
	Sharing(ctx context.Context, oid plumbing.Hash, expiresAt int64) (*Representation, error)
}

type LargeObject 

type LargeObject struct {
	OID            string `json:"oid"`
	CompressedSize int64  `json:"compressed_size"`
}

type MetadataDB 

type MetadataDB struct {
	*sql.DB
	// contains filtered or unexported fields
}

func NewMetadataDB 

func NewMetadataDB(db *sql.DB, rid int64) *MetadataDB

func (*MetadataDB) BatchEncodeCommit 

func (d *MetadataDB) BatchEncodeCommit(ctx context.Context, commits []*object.Commit) error

BatchEncodeCommit: batch encode commit to DB

func (*MetadataDB) BatchEncodeFragments 

func (d *MetadataDB) BatchEncodeFragments(ctx context.Context, fss []*object.Fragments) error

func (*MetadataDB) BatchEncodeTags 

func (d *MetadataDB) BatchEncodeTags(ctx context.Context, tags []*object.Tag) error

func (*MetadataDB) BatchEncodeTree 

func (d *MetadataDB) BatchEncodeTree(ctx context.Context, trees []*object.Tree) error

func (*MetadataDB) DecodeCommit 

func (d *MetadataDB) DecodeCommit(ctx context.Context, oid plumbing.Hash, b object.Backend) (*object.Commit, error)

func (*MetadataDB) DecodeTree 

func (d *MetadataDB) DecodeTree(ctx context.Context, oid plumbing.Hash, b object.Backend) (*object.Tree, error)

func (*MetadataDB) Encode 

func (d *MetadataDB) Encode(ctx context.Context, oid plumbing.Hash, e object.Encoder) error

func (*MetadataDB) EncodeCommit 

func (d *MetadataDB) EncodeCommit(ctx context.Context, cc *object.Commit) error

EncodeCommit: encode commit to DB

func (*MetadataDB) EncodeTree 

func (d *MetadataDB) EncodeTree(ctx context.Context, t *object.Tree) error

func (*MetadataDB) Fragments 

func (d *MetadataDB) Fragments(ctx context.Context, oid plumbing.Hash, b object.Backend) (*object.Fragments, error)

func (*MetadataDB) Object 

func (d *MetadataDB) Object(ctx context.Context, oid plumbing.Hash, b object.Backend) (any, error)

func (*MetadataDB) Tag 

func (d *MetadataDB) Tag(ctx context.Context, oid plumbing.Hash, b object.Backend) (*object.Tag, error)

type ODB 

type ODB struct {
	// contains filtered or unexported fields
}

func NewODB 

func NewODB(rid int64, root string, compressionALGO string, cdb CacheDB, mdb *MetadataDB, bucket oss.Bucket) (*ODB, error)

func (*ODB) Batch 

func (o *ODB) Batch(ctx context.Context, oids []plumbing.Hash, batchLimit int) error

Batch: batch upload objects

func (*ODB) BatchCommits 

func (o *ODB) BatchCommits(ctx context.Context, oids []plumbing.Hash) error

func (*ODB) BatchObjects 

func (o *ODB) BatchObjects(ctx context.Context, oids []plumbing.Hash) error

func (*ODB) BatchTrees 

func (o *ODB) BatchTrees(ctx context.Context, oids []plumbing.Hash) error

func (*ODB) Blob 

func (o *ODB) Blob(ctx context.Context, oid plumbing.Hash) (b *object.Blob, err error)

func (*ODB) Close 

func (o *ODB) Close() error

func (*ODB) Commit 

func (o *ODB) Commit(ctx context.Context, oid plumbing.Hash) (cc *object.Commit, err error)

func (*ODB) Encode 

func (o *ODB) Encode(ctx context.Context, e object.Encoder) (oid plumbing.Hash, err error)

EncodeEx: Store the object and update the Hash value of the object

func (*ODB) EncodeFast 

func (o *ODB) EncodeFast(ctx context.Context, e object.Encoder) (oid plumbing.Hash, err error)

EncodeFast: Store the object and update the Hash value of the object

func (*ODB) Fragments 

func (o *ODB) Fragments(ctx context.Context, oid plumbing.Hash) (*object.Fragments, error)

func (*ODB) HashFast 

func (o *ODB) HashFast(ctx context.Context, r io.Reader, size int64) (oid plumbing.Hash, err error)

func (*ODB) HashTo 

func (o *ODB) HashTo(ctx context.Context, r io.Reader, size int64) (oid plumbing.Hash, err error)

HashTo: Encode the read stream into a blob and upload it

func (*ODB) IsBinaryFast 

func (o *ODB) IsBinaryFast(ctx context.Context, oid plumbing.Hash) (bool, error)

func (*ODB) NewUnpacker 

func (o *ODB) NewUnpacker(entries uint32, metadata bool) (*backend.Unpacker, error)

func (*ODB) Objects 

func (o *ODB) Objects(ctx context.Context, oid plumbing.Hash) (a any, err error)

func (*ODB) Open 

func (o *ODB) Open(ctx context.Context, oid plumbing.Hash, start int64) (sr backend.SizeReader, err error)

func (*ODB) ParseRev 

func (o *ODB) ParseRev(ctx context.Context, oid plumbing.Hash) (a any, err error)

func (*ODB) ParseRevExhaustive 

func (o *ODB) ParseRevExhaustive(ctx context.Context, oid plumbing.Hash) (*object.Commit, error)

func (*ODB) Push 

func (o *ODB) Push(ctx context.Context, oid plumbing.Hash) error

func (*ODB) Reload 

func (o *ODB) Reload() error

Reload: reload odb

func (*ODB) Sharing 

func (o *ODB) Sharing(ctx context.Context, oid plumbing.Hash, expiresAt int64) (*Representation, error)

func (*ODB) Stat 

func (o *ODB) Stat(ctx context.Context, oid plumbing.Hash) (*oss.Stat, error)

func (*ODB) Tag 

func (o *ODB) Tag(ctx context.Context, oid plumbing.Hash) (*object.Tag, error)

func (*ODB) Tree 

func (o *ODB) Tree(ctx context.Context, oid plumbing.Hash) (t *object.Tree, err error)

func (*ODB) Unpack 

func (o *ODB) Unpack(ctx context.Context, r io.Reader, ss *OStats, validator Validator) (*Objects, error)

Unpack: 

FIXME: CRC64 verification has been temporarily stopped and may need to be restored later.

func (*ODB) WriteDirect 

func (o *ODB) WriteDirect(ctx context.Context, oid plumbing.Hash, r io.Reader, size int64) (int64, error)

Store directly to OSS Verify and upload to OSS Typically used for uploading larger binary files.

type OStats 

type OStats struct {
	M int
	B int
}

type Objects 

type Objects struct {
	Commits []plumbing.Hash // commits
	Trees   []plumbing.Hash // trees
	Objects []plumbing.Hash // fragments and tags
	Blobs   []plumbing.Hash // blobs
	Larges  []plumbing.Hash
}

type QuarantineDB 

type QuarantineDB struct {
	// contains filtered or unexported fields
}

func NewQuarantineDB 

func NewQuarantineDB(o *ODB, quarantineDir string) (*QuarantineDB, error)

func (*QuarantineDB) Close 

func (q *QuarantineDB) Close() error

func (*QuarantineDB) Commit 

func (q *QuarantineDB) Commit(ctx context.Context, oid plumbing.Hash) (cc *object.Commit, err error)

func (*QuarantineDB) Exists 

func (q *QuarantineDB) Exists(ctx context.Context, oid plumbing.Hash, meta bool) error

func (*QuarantineDB) Fragments 

func (q *QuarantineDB) Fragments(ctx context.Context, oid plumbing.Hash) (ff *object.Fragments, err error)

func (*QuarantineDB) ParseRev 

func (q *QuarantineDB) ParseRev(ctx context.Context, oid plumbing.Hash) (cc *object.Commit, isolated bool, err error)

func (*QuarantineDB) Tag 

func (q *QuarantineDB) Tag(ctx context.Context, oid plumbing.Hash) (tag *object.Tag, err error)

func (*QuarantineDB) Tree 

func (q *QuarantineDB) Tree(ctx context.Context, oid plumbing.Hash) (t *object.Tree, err error)

type Representation 

type Representation struct {
	Href      string
	Size      int64
	ExpiresAt int64
}

type StatObjectsResult 

type StatObjectsResult struct {
	Larges  []*LargeObject `json:"larges,omitempty"`
	Objects int            `json:"count"`
	Size    int64          `json:"size"`
}

func StatObjects 

func StatObjects(ctx context.Context, b oss.Bucket, rid int64, threshold int64) (*StatObjectsResult, error)

type Unpacker 

type Unpacker interface {
	Unpack(oid plumbing.Hash, r io.Reader, size uint32, metadata bool) (object.ObjectType, error)
	Close() error
}

type Unpackers 

type Unpackers struct {
	// contains filtered or unexported fields
}

func NewUnpackers 

func NewUnpackers(s *OStats, metadataRoot, blobRoot string) (*Unpackers, error)

func (*Unpackers) Close 

func (u *Unpackers) Close() error

func (*Unpackers) WriteTo 

func (u *Unpackers) WriteTo(oid plumbing.Hash, r io.Reader, size uint32) (object.ObjectType, error)

type Validator 

type Validator func(ctx context.Context, quarantineDir string, o *Objects) error

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.