deduplicator

Documentation

Index

• func AssignDatasetDataIdentityHashes(datasetData []data.Datum) ([]string, error)
• func GenerateIdentityHash(identityFields []string) (string, error)
• func IsVersionValid(version string) bool
• type BaseDeduplicator
  • func NewBaseDeduplicator(name string, version string, logger log.Logger, ...) (*BaseDeduplicator, error)
  • func (b *BaseDeduplicator) AddDatasetData(ctx context.Context, datasetData []data.Datum) error
  • func (b *BaseDeduplicator) DeduplicateDataset(ctx context.Context) error
  • func (b *BaseDeduplicator) DeleteDataset(ctx context.Context) error
  • func (b *BaseDeduplicator) Name() string
  • func (b *BaseDeduplicator) RegisterDataset(ctx context.Context) error
  • func (b *BaseDeduplicator) Version() string
• type BaseFactory
  • func NewBaseFactory(name string, version string) (*BaseFactory, error)
  • func (b *BaseFactory) CanDeduplicateDataset(dataset *upload.Upload) (bool, error)
  • func (b *BaseFactory) IsRegisteredWithDataset(dataset *upload.Upload) (bool, error)
  • func (b *BaseFactory) NewDeduplicatorForDataset(logger log.Logger, dataSession storeDEPRECATED.DataSession, ...) (data.Deduplicator, error)
  • func (b *BaseFactory) NewRegisteredDeduplicatorForDataset(logger log.Logger, dataSession storeDEPRECATED.DataSession, ...) (data.Deduplicator, error)
• type DelegateFactory
  • func NewDelegateFactory(factories []Factory) (*DelegateFactory, error)
  • func (d *DelegateFactory) CanDeduplicateDataset(dataset *upload.Upload) (bool, error)
  • func (d *DelegateFactory) IsRegisteredWithDataset(dataset *upload.Upload) (bool, error)
  • func (d *DelegateFactory) NewDeduplicatorForDataset(logger log.Logger, dataSession storeDEPRECATED.DataSession, ...) (data.Deduplicator, error)
  • func (d *DelegateFactory) NewRegisteredDeduplicatorForDataset(logger log.Logger, dataSession storeDEPRECATED.DataSession, ...) (data.Deduplicator, error)
• type Factory
  • func NewContinuousFactory() (Factory, error)
  • func NewHashDeactivateOldFactory() (Factory, error)
  • func NewTruncateFactory() (Factory, error)

Functions

func AssignDatasetDataIdentityHashes

func AssignDatasetDataIdentityHashes(datasetData []data.Datum) ([]string, error)

func GenerateIdentityHash

func GenerateIdentityHash(identityFields []string) (string, error)

func IsVersionValid

func IsVersionValid(version string) bool

Types

type BaseDeduplicator

type BaseDeduplicator struct {
	// contains filtered or unexported fields
}

func NewBaseDeduplicator

func NewBaseDeduplicator(name string, version string, logger log.Logger, dataSession storeDEPRECATED.DataSession, dataset *upload.Upload) (*BaseDeduplicator, error)

func (*BaseDeduplicator) AddDatasetData

func (b *BaseDeduplicator) AddDatasetData(ctx context.Context, datasetData []data.Datum) error

func (*BaseDeduplicator) DeduplicateDataset

func (b *BaseDeduplicator) DeduplicateDataset(ctx context.Context) error

func (*BaseDeduplicator) DeleteDataset

func (b *BaseDeduplicator) DeleteDataset(ctx context.Context) error

func (*BaseDeduplicator) Name

func (b *BaseDeduplicator) Name() string

func (*BaseDeduplicator) RegisterDataset

func (b *BaseDeduplicator) RegisterDataset(ctx context.Context) error

func (*BaseDeduplicator) Version

func (b *BaseDeduplicator) Version() string

type BaseFactory

type BaseFactory struct {
	Factory
	// contains filtered or unexported fields
}

func NewBaseFactory

func NewBaseFactory(name string, version string) (*BaseFactory, error)

func (*BaseFactory) CanDeduplicateDataset

func (b *BaseFactory) CanDeduplicateDataset(dataset *upload.Upload) (bool, error)

func (*BaseFactory) IsRegisteredWithDataset

func (b *BaseFactory) IsRegisteredWithDataset(dataset *upload.Upload) (bool, error)

func (*BaseFactory) NewDeduplicatorForDataset

func (b *BaseFactory) NewDeduplicatorForDataset(logger log.Logger, dataSession storeDEPRECATED.DataSession, dataset *upload.Upload) (data.Deduplicator, error)

func (*BaseFactory) NewRegisteredDeduplicatorForDataset

func (b *BaseFactory) NewRegisteredDeduplicatorForDataset(logger log.Logger, dataSession storeDEPRECATED.DataSession, dataset *upload.Upload) (data.Deduplicator, error)

type DelegateFactory

type DelegateFactory struct {
	// contains filtered or unexported fields
}

func NewDelegateFactory

func NewDelegateFactory(factories []Factory) (*DelegateFactory, error)

func (*DelegateFactory) CanDeduplicateDataset

func (d *DelegateFactory) CanDeduplicateDataset(dataset *upload.Upload) (bool, error)

func (*DelegateFactory) IsRegisteredWithDataset

func (d *DelegateFactory) IsRegisteredWithDataset(dataset *upload.Upload) (bool, error)

func (*DelegateFactory) NewDeduplicatorForDataset

func (d *DelegateFactory) NewDeduplicatorForDataset(logger log.Logger, dataSession storeDEPRECATED.DataSession, dataset *upload.Upload) (data.Deduplicator, error)

func (*DelegateFactory) NewRegisteredDeduplicatorForDataset

func (d *DelegateFactory) NewRegisteredDeduplicatorForDataset(logger log.Logger, dataSession storeDEPRECATED.DataSession, dataset *upload.Upload) (data.Deduplicator, error)

type Factory

type Factory interface {
	CanDeduplicateDataset(dataset *upload.Upload) (bool, error)
	NewDeduplicatorForDataset(logger log.Logger, dataSession storeDEPRECATED.DataSession, dataset *upload.Upload) (data.Deduplicator, error)

	IsRegisteredWithDataset(dataset *upload.Upload) (bool, error)
	NewRegisteredDeduplicatorForDataset(logger log.Logger, dataSession storeDEPRECATED.DataSession, dataset *upload.Upload) (data.Deduplicator, error)
}

func NewContinuousFactory

func NewContinuousFactory() (Factory, error)

func NewHashDeactivateOldFactory

func NewHashDeactivateOldFactory() (Factory, error)

func NewTruncateFactory

func NewTruncateFactory() (Factory, error)