model

package
v0.0.0-rc3 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Sep 25, 2024 License: Apache-2.0 Imports: 16 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	HourFormat  = "2006-01-02T15:04:05"
	DayFormat   = "2006-01-02"
	MonthFormat = "2006-01"
)
View Source 
const (
	LineSplitterNewLine    = LineSplitter("NEW_LINE")
	LineSplitterNoSplitter = LineSplitter("NO_SPLITTER")

	S3BackupModeErrorOnly    = BackupMode("ERROR_ONLY")
	S3BackupModeAllDocuments = BackupMode("ALL_DOCUMENTS")
	S3BackupModeNoBackup     = BackupMode("NO_BACKUP")

	ParsingFormatNative   = ParsingFormat("Native")
	ParsingFormatJSON     = ParsingFormat("JSON")
	ParsingFormatJSONLine = ParsingFormat("JSONL")
	ParsingFormatTSKV     = ParsingFormat("TSKV")
	ParsingFormatCSV      = ParsingFormat("CSV")
	ParsingFormatPROTO    = ParsingFormat("PROTO")
	ParsingFormatLine     = ParsingFormat("LINE")
	ParsingFormatORC      = ParsingFormat("ORC")
	ParsingFormatPARQUET  = ParsingFormat("PARQUET")
	ParsingFormatDebezium = ParsingFormat("Debezium")
	ParsingFormatRaw      = ParsingFormat("RAW")
)
View Source 
const (
	RotatorPartHour  = RotatorPartType("h")
	RotatorPartDay   = RotatorPartType("d")
	RotatorPartMonth = RotatorPartType("m")
)
View Source 
const (
	SerializationFormatAuto      = SerializationFormatName("Auto")
	SerializationFormatJSON      = SerializationFormatName("JSON")
	SerializationFormatDebezium  = SerializationFormatName("Debezium")
	SerializationFormatMirror    = SerializationFormatName("Mirror")
	SerializationFormatLbMirror  = SerializationFormatName("LbMirror")
	SerializationFormatNative    = SerializationFormatName("Native")
	SerializationFormatRawColumn = SerializationFormatName("RawColumn")

	ColumnNameParamName = "column_name"
)
View Source 
const (
	NewTask       = TaskStatus("New")
	ScheduledTask = TaskStatus("Scheduled")
	RunningTask   = TaskStatus("Running")
	CompletedTask = TaskStatus("Completed")
	FailedTask    = TaskStatus("Failed")
)
View Source 
const (
	Running = TransferStatus("Running")
	Stop    = TransferStatus("Stop")
	New     = TransferStatus("New")

	Scheduled = TransferStatus("Scheduled")
	Started   = TransferStatus("Started")
	Completed = TransferStatus("Completed")
	Failed    = TransferStatus("Failed")

	Stopping     = TransferStatus("Stopping")
	Creating     = TransferStatus("Creating")
	Deactivating = TransferStatus("Deactivating")
	Failing      = TransferStatus("Failing")
)
View Source 
const (
	ErrInvalidTmpPolicy = "invalid tmp policy: %w"
)
View Source 
const SystemLabelAsyncCH = SystemLabel("dt-async-ch")

SystemLabelAsyncCH is used to enable experimental asynchronous clickhouse snapshot sink

The only known value is "on", any other value means disabled.

View Source 
const SystemLabelMemThrottle = SystemLabel("dt-mem-throttle")

SystemLabelMemThrottle is used to enable/disable MemoryThrorrler middleware.

The only known value is "on", any other value means disabled.

Variables

View Source 
var (
	ErrRotatorConfigInvalidPartType      = errors.New("invalid part type")
	ErrRotatorConfigInvalidPartSize      = errors.New("part size should be positive (PartSize > 0)")
	ErrRotatorConfigInvalidKeepPartCount = errors.New("keep part count should be positive (KeepPartCount > 0)")
)
View Source 
var ActiveStatuses []TransferStatus
View Source 
var LocationEuropeMoscow *time.Location

LocationEuropeMoscow -- all time in rotator considered as Moscow time

Functions

func DestinationF 

func DestinationF(typ abstract.ProviderType) (func() Destination, bool)

func EnsureTmpPolicySupported 

func EnsureTmpPolicySupported(destination Destination, transfer *Transfer) error

func ExcludeViews 

func ExcludeViews(m abstract.TableMap) abstract.TableMap

func ExtractTimeCol 

func ExtractTimeCol(item abstract.ChangeItem, timeColumn string) time.Time

func FilteredMap 

func FilteredMap(m abstract.TableMap, incls ...abstract.Includeable) abstract.TableMap

FilteredMap filters IN-PLACE and returns its first argument

func FilteredTableList 

func FilteredTableList(storage abstract.Storage, transfer *Transfer) (abstract.TableMap, error)

func IsActiveStatus 

func IsActiveStatus(status TransferStatus) (bool, error)

func IsAppendOnlySource 

func IsAppendOnlySource(src Source) bool

func IsDefaultMirrorSource 

func IsDefaultMirrorSource(src Source) bool

func IsParseable 

func IsParseable(src Source) bool

func IsShardeableDestination 

func IsShardeableDestination(dst Destination) bool

func IsStrictSource 

func IsStrictSource(src Source) bool

func KnownDestinations 

func KnownDestinations() []string

func KnownSources 

func KnownSources() []string

func RegisterDestination 

func RegisterDestination(typ abstract.ProviderType, fac func() Destination)

RegisterDestination will add new destination factory for specific provider type this should be placed inside provide `init() func`

func RegisterSource 

func RegisterSource(typ abstract.ProviderType, fac func() Source)

RegisterSource will add new source factory for specific provider type this should be placed inside provider `init() func`

func SourceF 

func SourceF(typ abstract.ProviderType) (func() Source, bool)

Types

type Abstract2Source 

type Abstract2Source interface {
	IsAbstract2(Destination) bool
}

Abstract2Source if implemented we must try to create abstract2 source for them to be deleted in favor of `MakeDataProvider` method

type AggregatedProgress 

type AggregatedProgress struct {
	PartsCount          int64
	CompletedPartsCount int64
	ETARowsCount        int64
	CompletedRowsCount  int64
	TotalReadBytes      int64
	TotalDuration       time.Duration
	LastUpdateAt        time.Time
}

func NewAggregatedProgress 

func NewAggregatedProgress() *AggregatedProgress

func (*AggregatedProgress) Empty 

func (a *AggregatedProgress) Empty() bool

func (*AggregatedProgress) PartsPercent 

func (a *AggregatedProgress) PartsPercent() float64

func (*AggregatedProgress) RowsPercent 

func (a *AggregatedProgress) RowsPercent() float64

type AltName 

type AltName struct {
	From string
	To   string
}

type AppendOnlySource 

type AppendOnlySource interface {
	IsAppendOnly() bool
}

AppendOnlySource marks a source that can emit only InsertKind. It's queues (kafka/yds/eventhub/lb) with non-CDC parser Snapshot-only sources (gp/ch/yt/ydb) isn't AppendOnlySource, bcs of non-row events Queues without parser (mirror mode) are not AppendOnlySource

The contract for this interface (use IsAppendOnlySource() to check if source is appendOnly, to make it easy):

  • If source is not AppendOnlySource - it can emit updates/deletes
  • If source is AppendOnlySource - then we should call method IsAppendOnly() to determine can it emit updates/deletes (for queues-sources some parsers can emit updates/deletes (who carry CDC events), and other can't)

type AsyncPartSource 

type AsyncPartSource interface {
	IsAsyncShardPartsSource()
}

AsyncPartSource designates that source:

  1. Provides full and correct sequence of table loading events the sequence InitTableLoad -> [InitShardedTableLoad -> RowEvents -> DoneShardedTableLoad] x Shards -> DoneTableLoad for each table
  2. Can work with true AsyncSink where flush is only guaranteed on Done[Sharded]TableLoad event
  3. Provides correct PartID in ChangeItem if table is split into sharded parts.

type BackupMode 

type BackupMode string

type Batching 

type Batching struct {
	Enabled        bool
	Interval       time.Duration
	MaxChangeItems int
	MaxMessageSize int64
}

type BytesSize 

type BytesSize uint32

type CleanupType 

type CleanupType string
const (
	Drop            CleanupType = "Drop"
	Truncate        CleanupType = "Truncate"
	DisabledCleanup CleanupType = "Disabled"
)

func (CleanupType) IsValid 

func (ct CleanupType) IsValid() error

type Clusterable 

type Clusterable interface {
	MDBClusterID() string
}

type Dashboardeable 

type Dashboardeable interface {
	DashboardLink(linkedEndpoint EndpointParams) string
}

Dashboardeable will force specific dashboard link for endpoint pair see logfeller for example

type DataObjects 

type DataObjects struct {
	IncludeObjects []string `json:"includeObjects" yaml:"include_objects"`
}

func (*DataObjects) GetIncludeObjects 

func (o *DataObjects) GetIncludeObjects() []string

type DataTransformOptions 

type DataTransformOptions struct {
	CloudFunction         string
	ServiceAccountID      string
	NumberOfRetries       int64
	BufferSize            BytesSize
	BufferFlushInterval   time.Duration
	InvocationTimeout     time.Duration
	S3ErrorBucket         string
	BackupMode            BackupMode
	CloudFunctionsBaseURL string
	Headers               []HeaderValue
}

type DefaultMirrorSource 

type DefaultMirrorSource interface {
	IsDefaultMirror() bool
}

DefaultMirrorSource marks source as compatible with default mirror protocol (kafka/yds/eventhub)

type Destination 

type Destination interface {
	EndpointParams
	CleanupMode() CleanupType
	IsDestination()
}

func NewDestination 

func NewDestination(provider abstract.ProviderType, jsonStr string) (Destination, error)

type DestinationCompatibility 

type DestinationCompatibility interface {
	Compatible(dst Destination) error
}

DestinationCompatibility for source to check is it compatible with transfer destination

type EndpointParams 

type EndpointParams interface {
	GetProviderType() abstract.ProviderType
	Validate() error

	// WithDefaults sets default values for MISSING parameters of the endpoint
	WithDefaults()
}

type ExtraTransformableSource 

type ExtraTransformableSource interface {
	Source
	ExtraTransformers(ctx context.Context, transfer *Transfer, registry metrics.Registry) ([]abstract.Transformer, error)
}

type HackableTarget 

type HackableTarget interface {
	PreSnapshotHacks()
	PostSnapshotHacks()
}

HackableTarget hack target model and return rollback for that hack, used only by mysql and yt sources (for test mostly) plz don't use it.

type HeaderValue 

type HeaderValue struct {
	Key, Value string
}

type HostResolver 

type HostResolver interface {
	HostsNames() ([]string, error)
}

HostResolver returns a list of hosts to which network availability is required

type Includeable 

type Includeable interface {
	abstract.Includeable

	// FulfilledIncludes returns all include directives which are fulfilled by the given table
	FulfilledIncludes(tID abstract.TableID) []string

	// AllIncludes returns all include directives
	AllIncludes() []string
}

type IncrementalSource 

type IncrementalSource interface {
	IsIncremental()
	// SupportsStartCursorValue should return true if incremental source may use custom initial cursor value
	SupportsStartCursorValue() bool
}

IncrementalSource mark as enable incremental snapshot to transfer

type LegacyFillDependentFields 

type LegacyFillDependentFields interface {
	FillDependentFields(transfer *Transfer)
}

LegacyFillDependentFields for cp-dp backward compatibility, some fields were calculated on backend side before, so we preserve it

type LineSplitter 

type LineSplitter string

type ManagedEndpoint 

type ManagedEndpoint interface {
	EndpointParams
	ClusterID() string
}

type MockDestination 

type MockDestination struct {
	SinkerFactory func() abstract.Sinker
	Cleanup       CleanupType
}

func (*MockDestination) CleanupMode 

func (d *MockDestination) CleanupMode() CleanupType

func (*MockDestination) GetProviderType 

func (d *MockDestination) GetProviderType() abstract.ProviderType

func (*MockDestination) IsDestination 

func (d *MockDestination) IsDestination()

func (*MockDestination) MarshalJSON 

func (d *MockDestination) MarshalJSON() ([]byte, error)

func (*MockDestination) Transformer 

func (d *MockDestination) Transformer() map[string]string

func (*MockDestination) UnmarshalJSON 

func (d *MockDestination) UnmarshalJSON(data []byte) error

func (*MockDestination) Validate 

func (d *MockDestination) Validate() error

func (*MockDestination) WithDefaults 

func (d *MockDestination) WithDefaults()

type MockSource 

type MockSource struct {
	StorageFactory   func() abstract.Storage
	AllTablesFactory func() abstract.TableMap
}

func (*MockSource) GetName 

func (s *MockSource) GetName() string

func (*MockSource) GetProviderType 

func (s *MockSource) GetProviderType() abstract.ProviderType

func (MockSource) IsAsyncShardPartsSource 

func (MockSource) IsAsyncShardPartsSource()

func (MockSource) IsSource 

func (MockSource) IsSource()

func (*MockSource) Validate 

func (s *MockSource) Validate() error

func (*MockSource) WithDefaults 

func (s *MockSource) WithDefaults()

type MultiYtEnabled 

type MultiYtEnabled interface {
	MultiYtEnabled()
}

type OperationTablePart 

type OperationTablePart struct {
	OperationID   string
	Schema        string // Table schema or namespace
	Name          string // Table name
	Offset        uint64 // Table part offset
	Filter        string // Table part filter
	PartsCount    uint64 // Parts count for table
	PartIndex     uint64 // Index of this part in the table
	WorkerIndex   *int   // Worker index, that assigned to this part. If nil - worker not assigned yet.
	ETARows       uint64 // How much rows in this part
	CompletedRows uint64 // How much rows already copied
	ReadBytes     uint64 // How many bytes were read from the source
	Completed     bool   // Is this part already copied
}

func NewOperationTablePart 

func NewOperationTablePart() *OperationTablePart

func NewOperationTablePartFromDescription 

func NewOperationTablePartFromDescription(operationID string, description *abstract.TableDescription) *OperationTablePart

func (*OperationTablePart) CompletedPercent 

func (t *OperationTablePart) CompletedPercent() float64

func (*OperationTablePart) Copy 

func (t *OperationTablePart) Copy() *OperationTablePart

func (*OperationTablePart) Key 

func (t *OperationTablePart) Key() string

func (*OperationTablePart) Sharded 

func (t *OperationTablePart) Sharded() bool

func (*OperationTablePart) String 

func (t *OperationTablePart) String() string

func (*OperationTablePart) TableFQTN 

func (t *OperationTablePart) TableFQTN() string

func (*OperationTablePart) TableKey 

func (t *OperationTablePart) TableKey() string

func (*OperationTablePart) ToTableDescription 

func (t *OperationTablePart) ToTableDescription() *abstract.TableDescription

func (*OperationTablePart) ToTableID 

func (t *OperationTablePart) ToTableID() *abstract.TableID

type OperationWorker 

type OperationWorker struct {
	OperationID string
	WorkerIndex int
	Completed   bool
	Err         string
	Progress    *AggregatedProgress
}

func NewOperationWorker 

func NewOperationWorker() *OperationWorker

type OperationWorkflow 

type OperationWorkflow interface {
	OnStart(task *TransferOperation) error
	OnDone(task *TransferOperation) error
	OnError(task *TransferOperation, err error) error
}

type Parseable 

type Parseable interface {
	Parser() map[string]interface{}
}

Parseable provider unified access to parser config

type ParsingFormat 

type ParsingFormat string

type RotatorConfig 

type RotatorConfig struct {
	KeepPartCount     int
	PartType          RotatorPartType
	PartSize          int
	TimeColumn        string
	TableNameTemplate string
}

func (*RotatorConfig) Annotate 

func (p *RotatorConfig) Annotate(name string) string

Annotate is the default way to acquire rotated name based on the current time and structure settings: PartSize, PartType

func (*RotatorConfig) AnnotateWithTime 

func (p *RotatorConfig) AnnotateWithTime(name string, v time.Time) string

AnnotateWithTime produces rotated name with custom time `v` provided as argument

func (*RotatorConfig) AnnotateWithTimeFromColumn 

func (p *RotatorConfig) AnnotateWithTimeFromColumn(name string, item abstract.ChangeItem) string

AnnotateWithTimeFromColumn gives rotated name according to column with name TimeColumn as the time source

if column is absent, default Annotate method will be used

func (*RotatorConfig) BaseTime 

func (p *RotatorConfig) BaseTime() time.Time

BaseTime defines TTL of tables w.r.t. KeepPartCount window size parameter

func (*RotatorConfig) Next 

func (p *RotatorConfig) Next(name string) string

Next returns next rotated name with respect to current date

func (*RotatorConfig) NilWorkaround 

func (p *RotatorConfig) NilWorkaround() *RotatorConfig

TODO remove workaround when proper fix rolls out: YCDESIGN-1338 TM-1891

func (*RotatorConfig) ParseTime 

func (p *RotatorConfig) ParseTime(moscowTimeString string) (time.Time, error)

func (*RotatorConfig) Validate 

func (p *RotatorConfig) Validate() error

type RotatorPartType 

type RotatorPartType string

type Runtimeable 

type Runtimeable interface {
	Runtime() abstract.Runtime
}

Runtimeable will force specific runtime for endpoint see logfeller for example

type SecretString 

type SecretString string

type Serializable 

type Serializable interface {
	Serializer() (SerializationFormat, bool)
}

Serializable provider unified access to serializer config

type SerializationFormat 

type SerializationFormat struct {
	Name             SerializationFormatName
	Settings         map[string]string
	SettingsKV       [][2]string
	BatchingSettings *Batching
}

func (*SerializationFormat) Copy 

func (f *SerializationFormat) Copy() *SerializationFormat

type SerializationFormatName 

type SerializationFormatName string

type Serializeable 

type Serializeable interface {
	Params() string
	SetParams(string) error
}

type ShardeableDestination 

type ShardeableDestination interface {
	SupportSharding() bool
}

For default every destination is support sharding tables, but this interface can be used to override this behavior

type SnapshotParallelizationSupport 

type SnapshotParallelizationSupport interface {
	SupportMultiWorkers() bool
	SupportMultiThreads() bool
}

func GetSnapshotParallelizationSupport 

func GetSnapshotParallelizationSupport(params EndpointParams) SnapshotParallelizationSupport

type Source 

type Source interface {
	EndpointParams
	IsSource()
}

func NewSource 

func NewSource(provider abstract.ProviderType, jsonStr string) (Source, error)

type SourceCompatibility 

type SourceCompatibility interface {
	Compatible(src Source, transferType abstract.TransferType) error
}

SourceCompatibility for destination to check is it compatible with transfer source

type StrictSource 

type StrictSource interface {
	IsStrictSource()
}

type SystemLabel 

type SystemLabel string

SystemLabel contains transfer label names which are reserved to control hidden experimental transfer features.

Each SystemLabel value must be documented

type SystemTablesDependantDestination 

type SystemTablesDependantDestination interface {
	Destination
	ReliesOnSystemTablesTransferring() bool
}

type TLSMode 

type TLSMode string
const (
	DefaultTLS  TLSMode = "Default"
	EnabledTLS  TLSMode = "Enabled"
	DisabledTLS TLSMode = "Disabled"
)

type TableFilter 

type TableFilter func(tableID abstract.TableID) bool

type TaskStatus 

type TaskStatus string

func (TaskStatus) IsFinal 

func (s TaskStatus) IsFinal() bool

type TmpPolicyConfig 

type TmpPolicyConfig struct {
	Suffix string
	// contains filtered or unexported fields
}

func NewTmpPolicyConfig 

func NewTmpPolicyConfig(suffix string, include TableFilter) *TmpPolicyConfig

func (*TmpPolicyConfig) BuildSuffix 

func (c *TmpPolicyConfig) BuildSuffix(transferID string) string

func (*TmpPolicyConfig) Include 

func (c *TmpPolicyConfig) Include(tableID abstract.TableID) bool

func (*TmpPolicyConfig) WithInclude 

func (c *TmpPolicyConfig) WithInclude(include TableFilter) *TmpPolicyConfig

type TmpPolicyProvider 

type TmpPolicyProvider interface {
	//Common naive tmp policy: implemented via middleware which one just rename change items during snapshot and call sink method Move(tmp_table -> orig_table) on DoneLoadTable event.
	//Sharded snapshots are not supported in this mode.
	EnsureTmpPolicySupported() error
	//Some destinations have their own custom implementations with tmp policy logic (e.g. YT dynamic tables sink - UseStaticTablesOnSnapshot).
	//Custom policy is more preferable than common tmp policy.
	//If custom tmp policy is enabled we won`t drop tables on a destination on cleanup sinker stage.
	EnsureCustomTmpPolicySupported() error
}

type Transfer 

type Transfer struct {
	ID                string
	TransferName      string
	Description       string
	Labels            string
	Status            TransferStatus
	Type              abstract.TransferType
	Runtime           abstract.Runtime
	Src               Source
	Dst               Destination
	RegularSnapshot   *abstract.RegularSnapshot
	Transformation    *Transformation
	DataObjects       *DataObjects
	TypeSystemVersion int
	TmpPolicy         *TmpPolicyConfig

	// TODO: remove
	FolderID string
	CloudID  string
	Author   string
}

func (*Transfer) AddExtraTransformer 

func (f *Transfer) AddExtraTransformer(transformer abstract.Transformer) error

func (*Transfer) CanReloadFromState 

func (f *Transfer) CanReloadFromState() bool

func (*Transfer) Copy 

func (f *Transfer) Copy(name string) Transfer

func (*Transfer) CurrentJobIndex 

func (f *Transfer) CurrentJobIndex() int

func (*Transfer) DataObjectsFromJSON 

func (f *Transfer) DataObjectsFromJSON(objects string) error

func (*Transfer) DataObjectsJSON 

func (f *Transfer) DataObjectsJSON() (string, error)

func (*Transfer) DstJSON 

func (f *Transfer) DstJSON() string

func (*Transfer) DstType 

func (f *Transfer) DstType() abstract.ProviderType

func (*Transfer) FillDependentFields 

func (f *Transfer) FillDependentFields()

FillDependentFields

Should be called every time when new transfer object created For now, it's called: - in repository, when building transfer object from the db - in transitive uploading, when creating synthetic transfer - in e2e-tests, when building transfer object from endpoints (helpers: MakeTransfer/InitSrcDst)

func (*Transfer) FilterObjects 

func (f *Transfer) FilterObjects(result abstract.TableMap) (abstract.TableMap, error)

func (*Transfer) HasExtraTransformation 

func (f *Transfer) HasExtraTransformation() bool

func (*Transfer) HasPublicTransformation 

func (f *Transfer) HasPublicTransformation() bool

func (*Transfer) HasTransformation 

func (f *Transfer) HasTransformation() bool

func (*Transfer) Include 

func (f *Transfer) Include(tID abstract.TableID) bool

func (*Transfer) IncludeTableList 

func (f *Transfer) IncludeTableList() ([]abstract.TableID, error)

func (*Transfer) IncrementOnly 

func (f *Transfer) IncrementOnly() bool

func (*Transfer) IsAbstract2 

func (f *Transfer) IsAbstract2() bool

func (*Transfer) IsAsyncCHExp 

func (f *Transfer) IsAsyncCHExp() bool

func (*Transfer) IsIncremental 

func (f *Transfer) IsIncremental() bool

func (*Transfer) IsMain 

func (f *Transfer) IsMain() bool

func (*Transfer) IsSharded 

func (f *Transfer) IsSharded() bool

func (*Transfer) IsTransitional 

func (f *Transfer) IsTransitional() bool

IsTransitional show transfer that used by kostya and burn our pukans

func (*Transfer) LabelsRaw 

func (f *Transfer) LabelsRaw() string

func (*Transfer) ParallelismParams 

func (f *Transfer) ParallelismParams() *abstract.ShardUploadParams

func (*Transfer) RegularSnapshotEnabled 

func (f *Transfer) RegularSnapshotEnabled() bool

func (*Transfer) RegularSnapshotJSON 

func (f *Transfer) RegularSnapshotJSON() []byte

func (*Transfer) RuntimeType 

func (f *Transfer) RuntimeType() string

func (*Transfer) SnapshotOnly 

func (f *Transfer) SnapshotOnly() bool

func (*Transfer) SrcJSON 

func (f *Transfer) SrcJSON() string

func (*Transfer) SrcType 

func (f *Transfer) SrcType() abstract.ProviderType

func (*Transfer) SystemLabel 

func (f *Transfer) SystemLabel(name SystemLabel) (string, error)

SystemLabel method is used to access system labels for transfer. System labels are special reserved labels which are used to control some hidden experimental transfer features

func (*Transfer) TransformationConfigs 

func (f *Transfer) TransformationConfigs() []transformers_registry.Transformer

func (*Transfer) TransformationFromJSON 

func (f *Transfer) TransformationFromJSON(value string) error

func (*Transfer) TransformationJSON 

func (f *Transfer) TransformationJSON() ([]byte, error)

func (*Transfer) TransformationMiddleware 

func (f *Transfer) TransformationMiddleware() (abstract.SinkOption, error)

func (*Transfer) Validate 

func (f *Transfer) Validate() error

func (*Transfer) ValidateDataObjects 

func (f *Transfer) ValidateDataObjects() error

func (*Transfer) WithDefault 

func (f *Transfer) WithDefault()

type TransferOperation 

type TransferOperation struct {
	OperationID string
	TransferID  string
	TaskType    abstract.TaskType
	Status      TaskStatus
	Params      any // TODO: interface?
	Runtime     abstract.Runtime
	Progress    *AggregatedProgress
	Author      string
	PingedAt    time.Time
}

func (*TransferOperation) AggregatedProgress 

func (t *TransferOperation) AggregatedProgress() *AggregatedProgress

func (*TransferOperation) IsDone 

func (t *TransferOperation) IsDone() bool

func (*TransferOperation) NormalisedID 

func (t *TransferOperation) NormalisedID() string

func (*TransferOperation) String 

func (t *TransferOperation) String() string

type TransferStatus 

type TransferStatus string

type Transformation 

type Transformation struct {
	Transformers      *transformer.Transformers
	ExtraTransformers []abstract.Transformer
	Executor          abstract.Transformation
	RuntimeJobIndex   int
}

func MakeTransformationFromJSON 

func MakeTransformationFromJSON(value string) (*Transformation, error)

func (Transformation) Validate 

func (t Transformation) Validate() error

type TransitionalEndpoint 

type TransitionalEndpoint interface {
	TransitionalWith(right TransitionalEndpoint) bool
}

TransitionalEndpoint Mark endpoint as type as transitional, so it could be opt-out-ed on snapshot

type UnderlayOnlyEndpoint 

type UnderlayOnlyEndpoint interface {
	IsUnderlayOnlyEndpoint()
}

UnderlayOnlyEndpoint marks endpoint as available only via our service underlay network

type WithConnectionID 

type WithConnectionID interface {
	GetConnectionID() string
}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.