onboard

Documentation

Index

• Constants
• Variables
• func CreateCommitMetadata(inv block.Inventory, stats Stats, prefixes []string) catalog.Metadata
• func NewValueToEntryIterator(it *InventoryIterator, progress *cmdutils.Progress) catalog.EntryIterator
• type CatalogRepoActions
  • func NewCatalogRepoActions(config *Config, logger logging.Logger) *CatalogRepoActions
  • func (c *CatalogRepoActions) ApplyImport(ctx context.Context, it Iterator, _ bool) (*Stats, error)
  • func (c *CatalogRepoActions) Commit(ctx context.Context, commitMsg string, metadata catalog.Metadata) (string, error)
  • func (c *CatalogRepoActions) Init(ctx context.Context, baseCommit graveler.CommitID) error
  • func (c *CatalogRepoActions) Progress() []*cmdutils.Progress
• type Config
• type EntryCatalog
• type ImportObject
• type Importer
  • func CreateImporter(ctx context.Context, logger logging.Logger, config *Config) (importer *Importer, err error)
  • func (s *Importer) Import(ctx context.Context, dryRun bool) (*Stats, error)
  • func (s *Importer) Progress() []*cmdutils.Progress
• type InventoryDiff
• type InventoryIterator
  • func NewInventoryIterator(it block.InventoryIterator) *InventoryIterator
  • func (s *InventoryIterator) Get() ImportObject
• type Iterator
• type RepoActions
• type Stats

Constants

const (
	CommitMsgTemplate       = "Import from %s"
	DefaultImportBranchName = "import-from-inventory"
)

Variables

var (
	ErrIteratorNotSeekable = errors.New("can't seek on inventory iterator")
)

var ErrNoMetaRange = errors.New("nothing to commit - meta-range wasn't created")

var ErrNotSeekable = errors.New("iterator isn't seekable")

var ErrWrongIterator = errors.New("rocksCatalogRepoActions can only accept InventoryIterator")

Functions

func CreateCommitMetadata

func CreateCommitMetadata(inv block.Inventory, stats Stats, prefixes []string) catalog.Metadata

func NewValueToEntryIterator

func NewValueToEntryIterator(it *InventoryIterator, progress *cmdutils.Progress) catalog.EntryIterator

Types

type CatalogRepoActions

type CatalogRepoActions struct {
	// contains filtered or unexported fields
}

CatalogRepoActions is in-charge of importing data to lakeFS with Rocks implementation

func NewCatalogRepoActions

func NewCatalogRepoActions(config *Config, logger logging.Logger) *CatalogRepoActions

func (*CatalogRepoActions) ApplyImport

func (c *CatalogRepoActions) ApplyImport(ctx context.Context, it Iterator, _ bool) (*Stats, error)

func (*CatalogRepoActions) Commit

func (c *CatalogRepoActions) Commit(ctx context.Context, commitMsg string, metadata catalog.Metadata) (string, error)

func (*CatalogRepoActions) Init

func (c *CatalogRepoActions) Init(ctx context.Context, baseCommit graveler.CommitID) error

func (*CatalogRepoActions) Progress

func (c *CatalogRepoActions) Progress() []*cmdutils.Progress

type Config

type Config struct {
	CommitUsername     string
	InventoryURL       string
	RepositoryID       graveler.RepositoryID
	DefaultBranchID    graveler.BranchID
	InventoryGenerator block.InventoryGenerator
	Store              EntryCatalog
	CatalogActions     RepoActions
	KeyPrefixes        []string

	// BaseCommit is available only for import-plumbing command
	BaseCommit graveler.CommitID
}

type EntryCatalog

type EntryCatalog interface {
	WriteMetaRange(ctx context.Context, repositoryID graveler.RepositoryID, it graveler.ValueIterator) (*graveler.MetaRangeID, error)
	AddCommitToBranchHead(ctx context.Context, repositoryID graveler.RepositoryID, branchID graveler.BranchID, commit graveler.Commit) (graveler.CommitID, error)
	List(ctx context.Context, repositoryID graveler.RepositoryID, ref graveler.Ref) (graveler.ValueIterator, error)
	AddCommit(ctx context.Context, repositoryID graveler.RepositoryID, commit graveler.Commit) (graveler.CommitID, error)
	UpdateBranch(ctx context.Context, repositoryID graveler.RepositoryID, branchID graveler.BranchID, ref graveler.Ref) (*graveler.Branch, error)
	GetBranch(ctx context.Context, repositoryID graveler.RepositoryID, branchID graveler.BranchID) (*graveler.Branch, error)
	CreateBranch(ctx context.Context, repositoryID graveler.RepositoryID, branchID graveler.BranchID, ref graveler.Ref) (*graveler.Branch, error)
}

EntryCatalog is a facet for a catalog.Store

type ImportObject

type ImportObject struct {
	Obj       block.InventoryObject
	IsDeleted bool
	IsChanged bool
}

type Importer

type Importer struct {
	CatalogActions RepoActions
	// contains filtered or unexported fields
}

func CreateImporter

func CreateImporter(ctx context.Context, logger logging.Logger, config *Config) (importer *Importer, err error)

func (*Importer) Import

func (s *Importer) Import(ctx context.Context, dryRun bool) (*Stats, error)

func (*Importer) Progress

func (s *Importer) Progress() []*cmdutils.Progress

type InventoryDiff

type InventoryDiff struct {
	DryRun               bool
	AddedOrChanged       []block.InventoryObject
	Deleted              []block.InventoryObject
	PreviousInventoryURL string
	PreviousImportDate   time.Time
}

type InventoryIterator

type InventoryIterator struct {
	block.InventoryIterator
}

onboard.InventoryIterator reads from block.InventoryIterator and converts the objects to ImportObject

func NewInventoryIterator

func NewInventoryIterator(it block.InventoryIterator) *InventoryIterator

func (*InventoryIterator) Get

func (s *InventoryIterator) Get() ImportObject

type Iterator

type Iterator interface {
	cmdutils.ProgressReporter
	Next() bool
	Err() error
	Get() ImportObject
}

type RepoActions

type RepoActions interface {
	cmdutils.ProgressReporter
	ApplyImport(ctx context.Context, it Iterator, dryRun bool) (*Stats, error)
	Init(ctx context.Context, baseCommit graveler.CommitID) error
	Commit(ctx context.Context, commitMsg string, metadata catalog.Metadata) (commitRef string, err error)
}

type Stats

type Stats struct {
	AddedOrChanged     int
	DryRun             bool
	CommitRef          string
	PreviousImportDate time.Time
}