specs

package
v1.25.1 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jan 15, 2023 License: MPL-2.0 Imports: 9 Imported by: 20

Documentation

Overview

Package specs specs for source and destination plugins including parsers and readers.

Index

Constants

This section is empty.

Variables

View Source 
var AllBackendNames = [...]string{
	BackendNone:  "none",
	BackendLocal: "local",
}
View Source 
var AllBackends = Backends{BackendLocal}
View Source 
var AllSchedulerNames = [...]string{
	SchedulerDFS:        "dfs",
	SchedulerRoundRobin: "round-robin",
}
View Source 
var AllSchedulers = Schedulers{SchedulerDFS, SchedulerRoundRobin}

Functions

func SpecUnmarshalYamlStrict added in v0.0.4 

func SpecUnmarshalYamlStrict(b []byte, spec *Spec) error

func UnmarshalJSONStrict added in v0.5.0 

func UnmarshalJSONStrict(b []byte, out any) error

Types

type Backend added in v1.24.0 

type Backend int
const (
	BackendNone Backend = iota
	BackendLocal
)

func BackendFromString added in v1.24.0 

func BackendFromString(s string) (Backend, error)

func (Backend) MarshalJSON added in v1.24.0 

func (s Backend) MarshalJSON() ([]byte, error)

func (Backend) String added in v1.24.0 

func (s Backend) String() string

func (*Backend) UnmarshalJSON added in v1.24.0 

func (s *Backend) UnmarshalJSON(data []byte) (err error)

type Backends added in v1.24.0 

type Backends []Backend

func (Backends) String added in v1.24.0 

func (s Backends) String() string

type Destination added in v0.0.4 

type Destination struct {
	Name           string    `json:"name,omitempty"`
	Version        string    `json:"version,omitempty"`
	Path           string    `json:"path,omitempty"`
	Registry       Registry  `json:"registry,omitempty"`
	WriteMode      WriteMode `json:"write_mode,omitempty"`
	BatchSize      int       `json:"batch_size,omitempty"`
	BatchSizeBytes int       `json:"batch_size_bytes,omitempty"`
	Spec           any       `json:"spec,omitempty"`
}

func (*Destination) SetDefaults added in v0.0.4 

func (d *Destination) SetDefaults(defaultBatchSize, defaultBatchSizeBytes int)

func (*Destination) UnmarshalSpec added in v0.0.4 

func (d *Destination) UnmarshalSpec(out any) error

func (*Destination) Validate added in v0.9.2 

func (d *Destination) Validate() error

type Kind added in v0.0.4 

type Kind int
const (
	KindSource Kind = iota
	KindDestination
)

func KindFromString added in v0.0.4 

func KindFromString(s string) (Kind, error)

func (Kind) MarshalJSON added in v0.0.4 

func (k Kind) MarshalJSON() ([]byte, error)

func (Kind) String added in v0.0.4 

func (k Kind) String() string

func (*Kind) UnmarshalJSON added in v0.0.4 

func (k *Kind) UnmarshalJSON(data []byte) (err error)

type Registry added in v0.0.4 

type Registry int
const (
	RegistryGithub Registry = iota
	RegistryLocal
	RegistryGrpc
)

func RegistryFromString added in v0.0.4 

func RegistryFromString(s string) (Registry, error)

func (Registry) MarshalJSON added in v0.0.4 

func (r Registry) MarshalJSON() ([]byte, error)

func (Registry) String added in v0.0.4 

func (r Registry) String() string

func (*Registry) UnmarshalJSON added in v0.0.4 

func (r *Registry) UnmarshalJSON(data []byte) (err error)

type Scheduler added in v1.19.0 

type Scheduler int
const (
	SchedulerDFS Scheduler = iota
	SchedulerRoundRobin
)

func SchedulerFromString added in v1.19.0 

func SchedulerFromString(s string) (Scheduler, error)

func (Scheduler) MarshalJSON added in v1.19.0 

func (s Scheduler) MarshalJSON() ([]byte, error)

func (Scheduler) String added in v1.19.0 

func (s Scheduler) String() string

func (*Scheduler) UnmarshalJSON added in v1.19.0 

func (s *Scheduler) UnmarshalJSON(data []byte) (err error)

type Schedulers added in v1.19.0 

type Schedulers []Scheduler

func (Schedulers) String added in v1.19.0 

func (s Schedulers) String() string

type Source added in v0.0.4 

type Source struct {
	// Name of the source plugin to use
	Name string `json:"name,omitempty"`
	// Version of the source plugin to use
	Version string `json:"version,omitempty"`
	// Path is the canonical path to the source plugin in a given registry
	// For example:
	// in github the path will be: org/repo
	// For the local registry the path will be the path to the binary: ./path/to/binary
	// For the gRPC registry the path will be the address of the gRPC server: host:port
	Path string `json:"path,omitempty"`
	// Registry can be github,local,grpc.
	Registry            Registry `json:"registry,omitempty"`
	Concurrency         uint64   `json:"concurrency,omitempty"`
	TableConcurrency    uint64   `json:"table_concurrency,omitempty"`    // deprecated: use Concurrency instead
	ResourceConcurrency uint64   `json:"resource_concurrency,omitempty"` // deprecated: use Concurrency instead
	// Tables to sync from the source plugin
	Tables []string `json:"tables,omitempty"`
	// SkipTables defines tables to skip when syncing data. Useful if a glob pattern is used in Tables
	SkipTables []string `json:"skip_tables,omitempty"`
	// Destinations are the names of destination plugins to send sync data to
	Destinations []string `json:"destinations,omitempty"`

	// Backend is the name of the state backend to use
	Backend Backend `json:"backend,omitempty"`
	// Sync on
	OnlyIncrementalTables bool `json:"only_incremental_tables,omitempty"`
	// BackendSpec contains any backend-specific configuration
	BackendSpec any `json:"backend_spec,omitempty"`
	// Scheduler defines the scheduling algorithm that should be used to sync data
	Scheduler Scheduler `json:"scheduler,omitempty"`
	// Spec defines plugin specific configuration
	// This is different in every source plugin.
	Spec any `json:"spec,omitempty"`
}

Source is the spec for a source plugin

func (*Source) SetDefaults added in v0.0.4 

func (s *Source) SetDefaults()

func (*Source) UnmarshalBackendSpec added in v1.24.0 

func (s *Source) UnmarshalBackendSpec(out any) error

UnmarshalBackendSpec unmarshals the backend spec into the given interface

func (*Source) UnmarshalSpec added in v0.0.4 

func (s *Source) UnmarshalSpec(out any) error

UnmarshalSpec unmarshals the internal spec into the given interface

func (*Source) Validate added in v0.0.4 

func (s *Source) Validate() error

type Spec 

type Spec struct {
	Kind Kind `json:"kind"`
	Spec any  `json:"spec"`
}

func (*Spec) UnmarshalJSON added in v0.0.4 

func (s *Spec) UnmarshalJSON(data []byte) error

type SpecReader 

type SpecReader struct {
	Sources      map[string]*Source
	Destinations map[string]*Destination
}

func NewSpecReader 

func NewSpecReader(paths []string) (*SpecReader, error)

type WriteMode added in v0.0.4 

type WriteMode int
const (
	WriteModeOverwriteDeleteStale WriteMode = iota
	WriteModeOverwrite
	WriteModeAppend
)

func WriteModeFromString added in v0.0.4 

func WriteModeFromString(s string) (WriteMode, error)

func (WriteMode) MarshalJSON added in v0.0.4 

func (m WriteMode) MarshalJSON() ([]byte, error)

func (WriteMode) String added in v0.0.4 

func (m WriteMode) String() string

func (*WriteMode) UnmarshalJSON added in v0.0.4 

func (m *WriteMode) UnmarshalJSON(data []byte) (err error)

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.