badgeridx

Documentation

Index

• Constants
• type DB
  • func NewDB(options ...Option) (db *DB, err error)
  • func (db *DB) AddTask(id string, cancel context.CancelFunc)
  • func (db *DB) CancelReport(ctx context.Context, id string) error
  • func (db *DB) Close()
  • func (db *DB) CreateReport(ctx context.Context, req index.Request) (*schema.Report, error)
  • func (db *DB) Debug(ctx context.Context, req keyvalue.DebugRequest, ...) error
  • func (db *DB) Delete(ctx context.Context) error
  • func (db *DB) DeleteReport(ctx context.Context, id string) error
  • func (db *DB) DeleteTask(id string)
  • func (db *DB) FlushBatch()
  • func (db *DB) GetCdx(key string) (*schema.Cdx, error)
  • func (db *DB) GetFileInfo(_ context.Context, filename string) (*schema.FileInfo, error)
  • func (db *DB) GetReport(ctx context.Context, id string) (*schema.Report, error)
  • func (db *DB) GetStorageRef(ctx context.Context, warcId string) (string, error)
  • func (db *DB) Index(path string) error
  • func (db *DB) ListFileInfo(ctx context.Context, req index.Request, results chan<- index.FileInfoResponse) error
  • func (db *DB) ListReports(ctx context.Context, req index.Request, results chan<- index.ReportResponse) error
  • func (db *DB) ListStorageRef(ctx context.Context, req index.Request, results chan<- index.IdResponse) error
  • func (db *DB) Resolve(_ context.Context, warcId string) (storageRef string, err error)
  • func (db *DB) ResolvePath(filename string) (filePath string, err error)
  • func (db *DB) SaveReport(ctx context.Context, report *schema.Report) error
  • func (db *DB) Search(ctx context.Context, search index.Request, results chan<- index.CdxResponse) error
  • func (db *DB) Write(rec index.Record) error
• type Option
  • func WithBatchMaxSize(size int) Option
  • func WithBatchMaxWait(t time.Duration) Option
  • func WithCompression(c string) Option
  • func WithDatabase(db string) Option
  • func WithDir(d string) Option
  • func WithGcInterval(t time.Duration) Option
  • func WithIndexer(indexer index.Indexer) Option
  • func WithReadOnly(readOnly bool) Option
  • func WithoutBadgerLogging() Option
• type Options

Constants

const (
	SnappyCompression = "snappy"
	NoCompression     = "none"
	ZstdCompression   = "zstd"
)

Types

type DB

type DB struct {
	// IdIndex maps record id to storage ref
	IdIndex *badger.DB

	// FileIndex maps filepath to fileinfo
	FileIndex *badger.DB

	// CdxIndex maps cdx key to cdx record
	CdxIndex *badger.DB

	// ReportIndex maps report id to report
	ReportIndex *badger.DB
	// contains filtered or unexported fields
}

func NewDB

func NewDB(options ...Option) (db *DB, err error)

func (*DB) AddTask

func (db *DB) AddTask(id string, cancel context.CancelFunc)

func (*DB) CancelReport

func (db *DB) CancelReport(ctx context.Context, id string) error

func (*DB) Close

func (db *DB) Close()

Close stops the gc and batch workers and closes the index databases.

func (*DB) CreateReport

func (db *DB) CreateReport(ctx context.Context, req index.Request) (*schema.Report, error)

func (*DB) Debug

func (db *DB) Debug(ctx context.Context, req keyvalue.DebugRequest, results chan<- keyvalue.CdxResponse) error

func (*DB) Delete

func (db *DB) Delete(ctx context.Context) error

Delete removes all data from the database.

func (*DB) DeleteReport

func (db *DB) DeleteReport(ctx context.Context, id string) error

func (*DB) DeleteTask

func (db *DB) DeleteTask(id string)

func (*DB) FlushBatch

func (db *DB) FlushBatch()

FlushBatch collects all records in the batch channel and updates the id and cdx indices.

func (*DB) GetCdx

func (db *DB) GetCdx(key string) (*schema.Cdx, error)

func (*DB) GetFileInfo

func (db *DB) GetFileInfo(_ context.Context, filename string) (*schema.FileInfo, error)

func (*DB) GetReport

func (db *DB) GetReport(ctx context.Context, id string) (*schema.Report, error)

func (*DB) GetStorageRef

func (db *DB) GetStorageRef(ctx context.Context, warcId string) (string, error)

func (*DB) Index

func (db *DB) Index(path string) error

func (*DB) ListFileInfo

func (db *DB) ListFileInfo(ctx context.Context, req index.Request, results chan<- index.FileInfoResponse) error

func (*DB) ListReports

func (db *DB) ListReports(ctx context.Context, req index.Request, results chan<- index.ReportResponse) error

func (*DB) ListStorageRef

func (db *DB) ListStorageRef(ctx context.Context, req index.Request, results chan<- index.IdResponse) error

func (*DB) Resolve

func (db *DB) Resolve(_ context.Context, warcId string) (storageRef string, err error)

Resolve looks up warcId in the id index of the database and returns corresponding storageRef, or an error if not found.

func (*DB) ResolvePath

func (db *DB) ResolvePath(filename string) (filePath string, err error)

ResolvePath looks up filename in file index and returns the path field.

func (*DB) SaveReport

func (db *DB) SaveReport(ctx context.Context, report *schema.Report) error

func (*DB) Search

func (db *DB) Search(ctx context.Context, search index.Request, results chan<- index.CdxResponse) error

Search searches the index database

func (*DB) Write

func (db *DB) Write(rec index.Record) error

type Option

type Option func(opts *Options)

func WithBatchMaxSize

func WithBatchMaxSize(size int) Option

func WithBatchMaxWait

func WithBatchMaxWait(t time.Duration) Option

func WithCompression

func WithCompression(c string) Option

func WithDatabase

func WithDatabase(db string) Option

func WithDir

func WithDir(d string) Option

func WithGcInterval

func WithGcInterval(t time.Duration) Option

func WithIndexer

func WithIndexer(indexer index.Indexer) Option

func WithReadOnly

func WithReadOnly(readOnly bool) Option

func WithoutBadgerLogging

func WithoutBadgerLogging() Option

type Options

type Options struct {
	Compression  badgerOptions.CompressionType
	BatchMaxSize int
	BatchMaxWait time.Duration
	GcInterval   time.Duration
	Path         string
	ReadOnly     bool
	Database     string
	Index        index.Indexer
	Silent       bool
}