deduplicator

Documentation

Index

• func AssignDataSetDataIdentityHashes(dataSetData []data.Datum) ([]string, error)
• func GenerateIdentityHash(identityFields []string) (string, error)
• type BaseDeduplicator
  • func NewBaseDeduplicator(name string, version string, logger log.Logger, ...) (*BaseDeduplicator, error)
  • func (b *BaseDeduplicator) AddDataSetData(ctx context.Context, dataSetData []data.Datum) error
  • func (b *BaseDeduplicator) DeduplicateDataSet(ctx context.Context) error
  • func (b *BaseDeduplicator) DeleteDataSet(ctx context.Context) error
  • func (b *BaseDeduplicator) Name() string
  • func (b *BaseDeduplicator) RegisterDataSet(ctx context.Context) error
  • func (b *BaseDeduplicator) Version() string
• type BaseFactory
  • func NewBaseFactory(name string, version string) (*BaseFactory, error)
  • func (b *BaseFactory) CanDeduplicateDataSet(dataSet *upload.Upload) (bool, error)
  • func (b *BaseFactory) IsRegisteredWithDataSet(dataSet *upload.Upload) (bool, error)
  • func (b *BaseFactory) NewDeduplicatorForDataSet(logger log.Logger, dataSession storeDEPRECATED.DataSession, ...) (data.Deduplicator, error)
  • func (b *BaseFactory) NewRegisteredDeduplicatorForDataSet(logger log.Logger, dataSession storeDEPRECATED.DataSession, ...) (data.Deduplicator, error)
• type DelegateFactory
  • func NewDelegateFactory(factories []Factory) (*DelegateFactory, error)
  • func (d *DelegateFactory) CanDeduplicateDataSet(dataSet *upload.Upload) (bool, error)
  • func (d *DelegateFactory) IsRegisteredWithDataSet(dataSet *upload.Upload) (bool, error)
  • func (d *DelegateFactory) NewDeduplicatorForDataSet(logger log.Logger, dataSession storeDEPRECATED.DataSession, ...) (data.Deduplicator, error)
  • func (d *DelegateFactory) NewRegisteredDeduplicatorForDataSet(logger log.Logger, dataSession storeDEPRECATED.DataSession, ...) (data.Deduplicator, error)
• type Factory
  • func NewContinuousFactory() (Factory, error)
  • func NewContinuousOriginFactory() (Factory, error)
  • func NewHashDeactivateOldFactory() (Factory, error)
  • func NewTruncateFactory() (Factory, error)

Functions

func AssignDataSetDataIdentityHashes

func AssignDataSetDataIdentityHashes(dataSetData []data.Datum) ([]string, error)

func GenerateIdentityHash

func GenerateIdentityHash(identityFields []string) (string, error)

Types

type BaseDeduplicator

type BaseDeduplicator struct {
	// contains filtered or unexported fields
}

func NewBaseDeduplicator

func NewBaseDeduplicator(name string, version string, logger log.Logger, dataSession storeDEPRECATED.DataSession, dataSet *upload.Upload) (*BaseDeduplicator, error)

func (*BaseDeduplicator) AddDataSetData

func (b *BaseDeduplicator) AddDataSetData(ctx context.Context, dataSetData []data.Datum) error

func (*BaseDeduplicator) DeduplicateDataSet

func (b *BaseDeduplicator) DeduplicateDataSet(ctx context.Context) error

func (*BaseDeduplicator) DeleteDataSet

func (b *BaseDeduplicator) DeleteDataSet(ctx context.Context) error

func (*BaseDeduplicator) Name

func (b *BaseDeduplicator) Name() string

func (*BaseDeduplicator) RegisterDataSet

func (b *BaseDeduplicator) RegisterDataSet(ctx context.Context) error

func (*BaseDeduplicator) Version

func (b *BaseDeduplicator) Version() string

type BaseFactory

type BaseFactory struct {
	Factory
	// contains filtered or unexported fields
}

func NewBaseFactory

func NewBaseFactory(name string, version string) (*BaseFactory, error)

func (*BaseFactory) CanDeduplicateDataSet

func (b *BaseFactory) CanDeduplicateDataSet(dataSet *upload.Upload) (bool, error)

func (*BaseFactory) IsRegisteredWithDataSet

func (b *BaseFactory) IsRegisteredWithDataSet(dataSet *upload.Upload) (bool, error)

func (*BaseFactory) NewDeduplicatorForDataSet

func (b *BaseFactory) NewDeduplicatorForDataSet(logger log.Logger, dataSession storeDEPRECATED.DataSession, dataSet *upload.Upload) (data.Deduplicator, error)

func (*BaseFactory) NewRegisteredDeduplicatorForDataSet

func (b *BaseFactory) NewRegisteredDeduplicatorForDataSet(logger log.Logger, dataSession storeDEPRECATED.DataSession, dataSet *upload.Upload) (data.Deduplicator, error)

type DelegateFactory

type DelegateFactory struct {
	// contains filtered or unexported fields
}

func NewDelegateFactory

func NewDelegateFactory(factories []Factory) (*DelegateFactory, error)

func (*DelegateFactory) CanDeduplicateDataSet

func (d *DelegateFactory) CanDeduplicateDataSet(dataSet *upload.Upload) (bool, error)

func (*DelegateFactory) IsRegisteredWithDataSet

func (d *DelegateFactory) IsRegisteredWithDataSet(dataSet *upload.Upload) (bool, error)

func (*DelegateFactory) NewDeduplicatorForDataSet

func (d *DelegateFactory) NewDeduplicatorForDataSet(logger log.Logger, dataSession storeDEPRECATED.DataSession, dataSet *upload.Upload) (data.Deduplicator, error)

func (*DelegateFactory) NewRegisteredDeduplicatorForDataSet

func (d *DelegateFactory) NewRegisteredDeduplicatorForDataSet(logger log.Logger, dataSession storeDEPRECATED.DataSession, dataSet *upload.Upload) (data.Deduplicator, error)

type Factory

type Factory interface {
	CanDeduplicateDataSet(dataSet *upload.Upload) (bool, error)
	NewDeduplicatorForDataSet(logger log.Logger, dataSession storeDEPRECATED.DataSession, dataSet *upload.Upload) (data.Deduplicator, error)

	IsRegisteredWithDataSet(dataSet *upload.Upload) (bool, error)
	NewRegisteredDeduplicatorForDataSet(logger log.Logger, dataSession storeDEPRECATED.DataSession, dataSet *upload.Upload) (data.Deduplicator, error)
}

func NewContinuousFactory

func NewContinuousFactory() (Factory, error)

func NewContinuousOriginFactory

func NewContinuousOriginFactory() (Factory, error)

func NewHashDeactivateOldFactory

func NewHashDeactivateOldFactory() (Factory, error)

func NewTruncateFactory

func NewTruncateFactory() (Factory, error)