onboard

Documentation

Index

• Constants
• Variables
• func CreateCommitMetadata(inv block.Inventory, stats Stats, prefixes []string) catalog.Metadata
• func NewValueToEntryIterator(it *InventoryIterator, progress *cmdutils.Progress) catalog.EntryIterator
• type CatalogRepoActions
  • func NewCatalogRepoActions(config *Config, logger logging.Logger) *CatalogRepoActions
  • func (c *CatalogRepoActions) ApplyImport(ctx context.Context, it Iterator, _ bool) (*Stats, error)
  • func (c *CatalogRepoActions) Commit(ctx context.Context, commitMsg string, metadata catalog.Metadata) (string, error)
  • func (c *CatalogRepoActions) Init(ctx context.Context, baseCommit graveler.CommitID) error
  • func (c *CatalogRepoActions) Progress() []*cmdutils.Progress
• type Config
• type EntryCatalog
• type ImportObject
• type Importer
  • func CreateImporter(ctx context.Context, logger logging.Logger, config *Config) (importer *Importer, err error)
  • func (s *Importer) Import(ctx context.Context, dryRun bool) (*Stats, error)
  • func (s *Importer) Progress() []*cmdutils.Progress
• type InventoryDiff
• type InventoryIterator
  • func NewInventoryIterator(it block.InventoryIterator) *InventoryIterator
  • func (s *InventoryIterator) Get() ImportObject
• type Iterator
• type RepoActions
• type Stats

Constants

const (
	CommitMsgTemplate       = "Import from %s"
	DefaultImportBranchName = "import-from-inventory"
)

Variables

var (
	ErrIteratorNotSeekable = errors.New("can't seek on inventory iterator")
)

var ErrNoMetaRange = errors.New("nothing to commit - meta-range wasn't created")

var ErrNotSeekable = errors.New("iterator isn't seekable")

var ErrWrongIterator = errors.New("rocksCatalogRepoActions can only accept InventoryIterator")

Functions

func CreateCommitMetadata

func CreateCommitMetadata(inv block.Inventory, stats Stats, prefixes []string) catalog.Metadata

func NewValueToEntryIterator

func NewValueToEntryIterator(it *InventoryIterator, progress *cmdutils.Progress) catalog.EntryIterator

Types

type CatalogRepoActions

type CatalogRepoActions struct {
	// contains filtered or unexported fields
}

CatalogRepoActions is in-charge of importing data to lakeFS with Rocks implementation

func NewCatalogRepoActions

func NewCatalogRepoActions(config *Config, logger logging.Logger) *CatalogRepoActions

func (*CatalogRepoActions) ApplyImport

func (c *CatalogRepoActions) ApplyImport(ctx context.Context, it Iterator, _ bool) (*Stats, error)

func (*CatalogRepoActions) Commit

func (c *CatalogRepoActions) Commit(ctx context.Context, commitMsg string, metadata catalog.Metadata) (string, error)

func (*CatalogRepoActions) Init

func (c *CatalogRepoActions) Init(ctx context.Context, baseCommit graveler.CommitID) error

func (*CatalogRepoActions) Progress

func (c *CatalogRepoActions) Progress() []*cmdutils.Progress

type Config

type Config struct {
	CommitUsername     string
	InventoryURL       string
	RepositoryID       graveler.RepositoryID
	DefaultBranchID    graveler.BranchID
	InventoryGenerator block.InventoryGenerator
	Store              EntryCatalog
	CatalogActions     RepoActions
	KeyPrefixes        []string

	// BaseCommit is available only for import-plumbing command
	BaseCommit graveler.CommitID
}

type EntryCatalog

type EntryCatalog interface {
	GetRepository(ctx context.Context, repositoryID graveler.RepositoryID) (*graveler.RepositoryRecord, error)
	WriteMetaRangeByIterator(ctx context.Context, repository *graveler.RepositoryRecord, it graveler.ValueIterator) (*graveler.MetaRangeID, error)
	AddCommitToBranchHead(ctx context.Context, repository *graveler.RepositoryRecord, branchID graveler.BranchID, commit graveler.Commit) (graveler.CommitID, error)
	List(ctx context.Context, repository *graveler.RepositoryRecord, ref graveler.Ref) (graveler.ValueIterator, error)
	AddCommit(ctx context.Context, repository *graveler.RepositoryRecord, commit graveler.Commit) (graveler.CommitID, error)
	UpdateBranch(ctx context.Context, repository *graveler.RepositoryRecord, branchID graveler.BranchID, ref graveler.Ref) (*graveler.Branch, error)
	GetBranch(ctx context.Context, repository *graveler.RepositoryRecord, branchID graveler.BranchID) (*graveler.Branch, error)
	CreateBranch(ctx context.Context, repository *graveler.RepositoryRecord, branchID graveler.BranchID, ref graveler.Ref) (*graveler.Branch, error)
	GetCommit(ctx context.Context, repository *graveler.RepositoryRecord, commitID graveler.CommitID) (*graveler.Commit, error)
}

EntryCatalog is a facet for a catalog.Store

type ImportObject

type ImportObject struct {
	Obj       block.InventoryObject
	IsDeleted bool
	IsChanged bool
}

type Importer

type Importer struct {
	CatalogActions RepoActions
	// contains filtered or unexported fields
}

func CreateImporter

func CreateImporter(ctx context.Context, logger logging.Logger, config *Config) (importer *Importer, err error)

func (*Importer) Import

func (s *Importer) Import(ctx context.Context, dryRun bool) (*Stats, error)

func (*Importer) Progress

func (s *Importer) Progress() []*cmdutils.Progress

type InventoryDiff

type InventoryDiff struct {
	DryRun               bool
	AddedOrChanged       []block.InventoryObject
	Deleted              []block.InventoryObject
	PreviousInventoryURL string
	PreviousImportDate   time.Time
}

type InventoryIterator

type InventoryIterator struct {
	block.InventoryIterator
}

onboard.InventoryIterator reads from block.InventoryIterator and converts the objects to ImportObject

func NewInventoryIterator

func NewInventoryIterator(it block.InventoryIterator) *InventoryIterator

func (*InventoryIterator) Get

func (s *InventoryIterator) Get() ImportObject

type Iterator

type Iterator interface {
	cmdutils.ProgressReporter
	Next() bool
	Err() error
	Get() ImportObject
}

type RepoActions

type RepoActions interface {
	cmdutils.ProgressReporter
	ApplyImport(ctx context.Context, it Iterator, dryRun bool) (*Stats, error)
	Init(ctx context.Context, baseCommit graveler.CommitID) error
	Commit(ctx context.Context, commitMsg string, metadata catalog.Metadata) (commitRef string, err error)
}

type Stats

type Stats struct {
	AddedOrChanged     int
	DryRun             bool
	CommitRef          string
	PreviousImportDate time.Time
}