index

Documentation

Index

• Variables
• func NewInverted(dts ds.Batching) *invertedIndexImpl
• type FSIndexRepo
  • func NewFSRepo(baseDir string) (*FSIndexRepo, error)
  • func (l *FSIndexRepo) AddFullIndex(key shard.Key, index carindex.Index) (err error)
  • func (l *FSIndexRepo) DropFullIndex(key shard.Key) (dropped bool, err error)
  • func (l *FSIndexRepo) ForEach(f func(shard.Key) (bool, error)) error
  • func (l *FSIndexRepo) GetFullIndex(key shard.Key) (carindex.Index, error)
  • func (l *FSIndexRepo) Len() (int, error)
  • func (l *FSIndexRepo) Size() (uint64, error)
  • func (l *FSIndexRepo) StatFullIndex(key shard.Key) (Stat, error)
• type FullIndexRepo
• type Inverted
• type Manifest
• type ManifestKey
• type ManifestRepo
• type MemIndexRepo
  • func NewMemoryRepo() *MemIndexRepo
  • func (m *MemIndexRepo) AddFullIndex(key shard.Key, index index.Index) (err error)
  • func (m *MemIndexRepo) DropFullIndex(key shard.Key) (dropped bool, err error)
  • func (m *MemIndexRepo) ForEach(f func(shard.Key) (bool, error)) error
  • func (m *MemIndexRepo) GetFullIndex(key shard.Key) (idx index.Index, err error)
  • func (m *MemIndexRepo) Len() (int, error)
  • func (m *MemIndexRepo) Size() (uint64, error)
  • func (m *MemIndexRepo) StatFullIndex(key shard.Key) (Stat, error)
• type MultihashIterator
• type Repo
• type Stat

Variables

var ErrNotFound = errors.New("index not found")

Functions

func NewInverted

func NewInverted(dts ds.Batching) *invertedIndexImpl

NewInverted returns a new inverted index that uses `go-indexer-core` as it's storage backend. We use `go-indexer-core` as the backend here as it's been optimized to store (multihash -> Value) kind of data and supports bulk updates via context ID and metadata-deduplication which are useful properties for our use case here.

Types

type FSIndexRepo

type FSIndexRepo struct {
	// contains filtered or unexported fields
}

FSIndexRepo implements FullIndexRepo using the local file system to store the indices

func NewFSRepo

func NewFSRepo(baseDir string) (*FSIndexRepo, error)

NewFSRepo creates a new index repo that stores indices on the local filesystem with the given base directory as the root

func (*FSIndexRepo) AddFullIndex

func (l *FSIndexRepo) AddFullIndex(key shard.Key, index carindex.Index) (err error)

func (*FSIndexRepo) DropFullIndex

func (l *FSIndexRepo) DropFullIndex(key shard.Key) (dropped bool, err error)

func (*FSIndexRepo) ForEach

func (l *FSIndexRepo) ForEach(f func(shard.Key) (bool, error)) error

ForEach iterates over each index file to extract the key

func (*FSIndexRepo) GetFullIndex

func (l *FSIndexRepo) GetFullIndex(key shard.Key) (carindex.Index, error)

func (*FSIndexRepo) Len

func (l *FSIndexRepo) Len() (int, error)

Len counts all index files in the base path

func (*FSIndexRepo) Size

func (l *FSIndexRepo) Size() (uint64, error)

Size sums the size of all index files in the base path

func (*FSIndexRepo) StatFullIndex

func (l *FSIndexRepo) StatFullIndex(key shard.Key) (Stat, error)

type FullIndexRepo

type FullIndexRepo interface {
	// GetFullIndex returns the full index for the specified shard.
	GetFullIndex(key shard.Key) (index.Index, error)

	// AddFullIndex persists a full index for a shard.
	AddFullIndex(key shard.Key, index index.Index) error

	// DropFullIndex drops the full index for the specified shard. If the error
	// is nil, it returns whether an index was effectively dropped.
	DropFullIndex(key shard.Key) (dropped bool, err error)

	// StatFullIndex stats a full index.
	StatFullIndex(key shard.Key) (Stat, error)

	// Len returns the number of indices in the repo.
	Len() (int, error)

	// ForEach calls the callback with the key for each index.
	//
	// Returning true from the callback will continue the traversal.
	// Returning false will terminate the traversal.
	//
	// A non-nil error will abort the traversal, and the error will be
	// propagated to the caller.
	ForEach(func(shard.Key) (bool, error)) error

	// Size returns the size of the repo in bytes.
	Size() (uint64, error)
}

type Inverted

type Inverted interface {
	// AddMultihashesForShard adds a (multihash -> shard key) mapping for all multihashes returned by the given MultihashIterator.
	AddMultihashesForShard(ctx context.Context, mhIter MultihashIterator, s shard.Key) error
	// GetShardsForMultihash returns keys for all the shards that has the given multihash.
	GetShardsForMultihash(ctx context.Context, h multihash.Multihash) ([]shard.Key, error)
}

Inverted is the top-level inverted index that maps a multihash to all the shards it is present in.

type Manifest

type Manifest interface {
	// Contains checks whether a given CID is contained in the manifest.
	Contains(c cid.Cid) (bool, error)

	// Len returns the count of entries this manifest has.
	Len() (l int64, err error)

	// ForEach traverses the manifest using an visitor pattern. The supplied
	// callback will be called for each manifest entry, in no particular order.
	//
	// Returning true from the callback will continue the traversal.
	// Returning false will terminate the traversal.
	//
	// A non-nil error will abort the traversal, and the error will be
	// propagated to the caller.
	ForEach(func(c cid.Cid) (ok bool, err error)) error
}

Manifest are sets of CIDs with no offset indication.

type ManifestKey

type ManifestKey struct {
	Shard      shard.Key
	GenRule    string
	GenVersion uint
}

ManifestKey identifies a manifest. It is a triple that can act as a composite key, comprising the shard key and generation metadata.

type ManifestRepo

type ManifestRepo interface {
	// ListManifests returns the available manifests for a given shard,
	// identified by their ManifestKey.
	ListManifests(key shard.Key) ([]ManifestKey, error)

	// GetManifest returns the Manifest identified by a given ManifestKey.
	GetManifest(key ManifestKey) (Manifest, error)

	// AddManifest adds a Manifest to the ManifestRepo.
	AddManifest(key ManifestKey, manifest Manifest) error

	// DropManifest drops a Manifest from the ManifestRepo.
	DropManifest(key ManifestKey) (bool, error)

	// StatManifest stats a Manifest.
	StatManifest(key ManifestKey) (Stat, error)
}

TODO unimplemented.

type MemIndexRepo

type MemIndexRepo struct {
	// contains filtered or unexported fields
}

MemIndexRepo implements FullIndexRepo with an in-memory map.

func NewMemoryRepo

func NewMemoryRepo() *MemIndexRepo

func (*MemIndexRepo) AddFullIndex

func (m *MemIndexRepo) AddFullIndex(key shard.Key, index index.Index) (err error)

func (*MemIndexRepo) DropFullIndex

func (m *MemIndexRepo) DropFullIndex(key shard.Key) (dropped bool, err error)

func (*MemIndexRepo) ForEach

func (m *MemIndexRepo) ForEach(f func(shard.Key) (bool, error)) error

func (*MemIndexRepo) GetFullIndex

func (m *MemIndexRepo) GetFullIndex(key shard.Key) (idx index.Index, err error)

func (*MemIndexRepo) Len

func (m *MemIndexRepo) Len() (int, error)

func (*MemIndexRepo) Size

func (m *MemIndexRepo) Size() (uint64, error)

func (*MemIndexRepo) StatFullIndex

func (m *MemIndexRepo) StatFullIndex(key shard.Key) (Stat, error)

type MultihashIterator

type MultihashIterator interface {
	ForEach(func(mh multihash.Multihash) error) error
}

type Repo

type Repo interface {
	FullIndexRepo
	ManifestRepo
}

Repo is the central index repository object that manages full indices and manifests.

type Stat

type Stat struct {
	Exists bool
	Size   uint64
}