specs

package
v0.40.2 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Dec 1, 2024 License: MPL-2.0 Imports: 19 Imported by: 0

Documentation

Overview

Package specs includes specs for source and destination plugins including parsers and readers.

Index

Constants

This section is empty.

Variables

View Source 
var (
	AllKinds = [...]string{
		KindSource:      "source",
		KindDestination: "destination",
		KindTransformer: "transformer",
	}
)
View Source 
var (
	AllMigrateModes = [...]string{
		MigrateModeSafe:   "safe",
		MigrateModeForced: "forced",
	}
)
View Source 
var (
	AllPKModes = [...]string{
		PKModeDefaultKeys: "default",
		PKModeCQID:        "cq-id-only",
	}
)
View Source 
var (
	AllRegistries = [...]string{
		RegistryUnset:      "",
		RegistryGitHub:     "github",
		RegistryLocal:      "local",
		RegistryGRPC:       "grpc",
		RegistryDocker:     "docker",
		RegistryCloudQuery: "cloudquery",
	}
)
View Source 
var (
	AllWriteModes = [...]string{
		WriteModeOverwriteDeleteStale: "overwrite-delete-stale",
		WriteModeOverwrite:            "overwrite",
		WriteModeAppend:               "append",
	}
)

Functions

func ReplaceVariables 

func ReplaceVariables(src string, variables Variables, shouldReplaceLocalhost bool) (string, error)

ReplaceVariables replaces variables starting with @@ in the src string with the values from the values from variables by dot notation. Example: @@plugins.aws.connection will be replaced with the value of variables.Plugins["aws"].Connection

func SpecUnmarshalYamlStrict 

func SpecUnmarshalYamlStrict(b []byte, spec *Spec) error

func WarnOnOutdatedVersions 

func WarnOnOutdatedVersions(ctx context.Context, p *managedplugin.PluginVersionWarner, sources []*Source, destinations []*Destination, transformers []*Transformer)

Types

type BackendOptions 

type BackendOptions struct {
	// The name of the table to store the key-value pairs for incremental progress.
	TableName string `json:"table_name,omitempty" jsonschema:"required,minLength=1"`

	// Connection string for the destination plugin.
	// Can be either `@@plugin.name.connection` or a fully-qualified gRPC connection string.
	Connection string `json:"connection,omitempty" jsonschema:"required,minLength=1"`
}

Backend options to be used in conjunction with incremental tables (stores the incremental progres)

func (*BackendOptions) PluginName 

func (b *BackendOptions) PluginName() string

PluginName returns the name of the plugin from the connection string variable.

Note that `Connection` gets string replaced with the actual connection value during the sync process, so calling this function will only work before the sync process starts.

type Destination 

type Destination struct {
	Metadata

	// Destination plugin write mode
	WriteMode WriteMode `json:"write_mode,omitempty" jsonschema:"default=overwrite-delete-stale"`

	// Destination plugin migrate mode
	MigrateMode MigrateMode `json:"migrate_mode,omitempty" jsonschema:"default=safe"`

	// Destination plugin PK mode
	PKMode PKMode `json:"pk_mode,omitempty" jsonschema:"default=default"`

	SyncGroupId string `json:"sync_group_id,omitempty"`

	SyncSummary bool `json:"send_sync_summary,omitempty"`

	// Transformers are the names of transformer plugins to send sync data through
	Transformers []string `json:"transformers,omitempty"`

	// Destination plugin own (nested) spec
	Spec map[string]any `json:"spec,omitempty"`
}

Destination plugin spec

func (*Destination) GetWarnings 

func (*Destination) GetWarnings() Warnings

func (*Destination) RenderedSyncGroupId 

func (d *Destination) RenderedSyncGroupId(t time.Time, sync_id string) string

func (*Destination) SetDefaults 

func (d *Destination) SetDefaults()

func (*Destination) UnmarshalSpec 

func (d *Destination) UnmarshalSpec(out any) error

func (*Destination) Validate 

func (d *Destination) Validate() error

type Kind 

type Kind int
const (
	KindSource Kind = iota
	KindDestination
	KindTransformer
)

func KindFromString 

func KindFromString(s string) (Kind, error)

func (Kind) JSONSchemaExtend 

func (Kind) JSONSchemaExtend(sc *jsonschema.Schema)

func (Kind) MarshalJSON 

func (k Kind) MarshalJSON() ([]byte, error)

func (Kind) String 

func (k Kind) String() string

func (*Kind) UnmarshalJSON 

func (k *Kind) UnmarshalJSON(data []byte) (err error)

type Metadata 

type Metadata struct {
	// Name of the plugin to use
	Name string `json:"name" jsonschema:"required,minLength=1"`

	// Version of the plugin to be used
	Version string `json:"version"`

	// Path is the canonical path to the plugin in a given registry
	// For example:
	// * for `registry: github` the `path` will be: `org/repo`
	// * for `registry: local` the `path` will be the path to the binary: `./path/to/binary`
	// * for `registry: grpc` the `path` will be the address of the gRPC server: `host:port`
	// * for `registry: cloudquery` the `path` will be: `team/name`
	Path string `json:"path" jsonschema:"required,minLength=1"`

	// Registry can be "", "github", "local", "grpc", "docker", "cloudquery"
	Registry Registry `json:"registry,omitempty" jsonschema:"default=cloudquery"`

	// DockerRegistryAuthToken is the token to use to authenticate with the docker registry
	DockerRegistryAuthToken string `json:"docker_registry_auth_token,omitempty"`
	// contains filtered or unexported fields
}

Spec part to define exact plugin: name, version & location.

func (Metadata) JSONSchemaExtend 

func (Metadata) JSONSchemaExtend(sc *jsonschema.Schema)

JSONSchemaExtend has to be in sync with Registry.NeedVersion

func (*Metadata) RegistryInferred 

func (m *Metadata) RegistryInferred() bool

func (*Metadata) SetDefaults 

func (m *Metadata) SetDefaults()

func (*Metadata) Validate 

func (m *Metadata) Validate() error

func (Metadata) VersionString 

func (m Metadata) VersionString() string

type MigrateMode 

type MigrateMode int
const (
	MigrateModeSafe MigrateMode = iota
	MigrateModeForced
)

func MigrateModeFromString 

func MigrateModeFromString(s string) (MigrateMode, error)

func (MigrateMode) JSONSchemaExtend 

func (MigrateMode) JSONSchemaExtend(sc *jsonschema.Schema)

func (MigrateMode) MarshalJSON 

func (m MigrateMode) MarshalJSON() ([]byte, error)

func (MigrateMode) String 

func (m MigrateMode) String() string

func (*MigrateMode) UnmarshalJSON 

func (m *MigrateMode) UnmarshalJSON(data []byte) (err error)

type PKMode 

type PKMode int
const (
	PKModeDefaultKeys PKMode = iota
	PKModeCQID
)

func PKModeFromString 

func PKModeFromString(s string) (PKMode, error)

func (PKMode) JSONSchemaExtend 

func (PKMode) JSONSchemaExtend(sc *jsonschema.Schema)

func (PKMode) MarshalJSON 

func (m PKMode) MarshalJSON() ([]byte, error)

func (PKMode) String 

func (m PKMode) String() string

func (*PKMode) UnmarshalJSON 

func (m *PKMode) UnmarshalJSON(data []byte) (err error)

type PluginVariables 

type PluginVariables struct {
	Connection string `json:"connection"`
}

type Registry 

type Registry int
const (
	RegistryUnset  Registry = iota
	RegistryGitHub          // deprecated
	RegistryLocal
	RegistryGRPC
	RegistryDocker
	RegistryCloudQuery
)

func RegistryFromString 

func RegistryFromString(s string) (Registry, error)

func (Registry) JSONSchemaExtend 

func (Registry) JSONSchemaExtend(sc *jsonschema.Schema)

func (Registry) MarshalJSON 

func (r Registry) MarshalJSON() ([]byte, error)

func (Registry) NeedVersion 

func (r Registry) NeedVersion() bool

NeedVersion has to be in sync with Metadata.JSONSchemaExtend

func (Registry) String 

func (r Registry) String() string

func (*Registry) UnmarshalJSON 

func (r *Registry) UnmarshalJSON(data []byte) (err error)

type Source 

type Source struct {
	Metadata

	// Tables to sync from the source plugin
	Tables []string `json:"tables,omitempty" jsonschema:"required,minItems=1,minLength=1"`
	// SkipTables defines tables to skip when syncing data. Useful if a glob pattern is used in Tables
	SkipTables []string `json:"skip_tables,omitempty" jsonschema:"minLength=1"`
	// SkipDependentTables changes the matching behavior with regard to dependent tables. If set to `false`, dependent tables will be included in the sync when their parents are matched, even if not explicitly included by the `tables` configuration.
	SkipDependentTables *bool `json:"skip_dependent_tables,omitempty" jsonschema:"default=true"`

	// Destinations are the names of destination plugins to send sync data to
	Destinations []string `json:"destinations,omitempty" jsonschema:"required,minItems=1,minLength=1"`

	// Optional Backend options for sync operation
	BackendOptions *BackendOptions `json:"backend_options,omitempty"`

	// Source plugin own (nested) spec
	Spec map[string]any `json:"spec,omitempty"`

	// DeterministicCQID is a flag that indicates whether the source plugin should generate a random UUID as the value of `_cq_id`
	// or whether it should calculate a UUID that is a hash of the primary keys (if they exist) or the entire resource.
	DeterministicCQID bool `json:"deterministic_cq_id,omitempty" jsonschema:"default=false"`

	// If specified this will spawn the plugin with `--otel-endpoint`
	OtelEndpoint string `json:"otel_endpoint,omitempty" jsonschema:"default="`
	// If specified this will spawn the plugin with `--otel-endpoint-insecure`
	OtelEndpointInsecure bool `json:"otel_endpoint_insecure,omitempty" jsonschema:"default=false"`
}

Source plugin spec

func (*Source) GetWarnings 

func (s *Source) GetWarnings() Warnings

GetWarnings returns a list of deprecated options that were used in the source config. This should be called before SetDefaults.

func (Source) JSONSchemaExtend 

func (Source) JSONSchemaExtend(sc *jsonschema.Schema)

func (*Source) SetDefaults 

func (s *Source) SetDefaults()

func (*Source) UnmarshalSpec 

func (s *Source) UnmarshalSpec(out any) error

UnmarshalSpec unmarshals the internal spec into the given interface

func (*Source) Validate 

func (s *Source) Validate() error

type Spec 

type Spec struct {
	// CloudQuery plugin kind
	Kind Kind `json:"kind" jsonschema:"required"`

	// CloudQuery plugin (top-level) spec
	Spec any `json:"spec" jsonschema:"required"`
}

func (Spec) JSONSchemaExtend 

func (Spec) JSONSchemaExtend(sc *jsonschema.Schema)

func (*Spec) UnmarshalJSON 

func (s *Spec) UnmarshalJSON(data []byte) error

type SpecReader 

type SpecReader struct {
	Sources      []*Source
	Destinations []*Destination
	Transformers []*Transformer
	// contains filtered or unexported fields
}

func NewRelaxedSpecReader 

func NewRelaxedSpecReader(paths []string) (*SpecReader, error)

func NewSpecReader 

func NewSpecReader(paths []string) (*SpecReader, error)

func (*SpecReader) GetDestinationByName 

func (r *SpecReader) GetDestinationByName(name string) *Destination

func (*SpecReader) GetDestinationNamesForSource 

func (r *SpecReader) GetDestinationNamesForSource(name string) []string

func (*SpecReader) GetDestinationWarningsByName 

func (r *SpecReader) GetDestinationWarningsByName(name string) Warnings

func (*SpecReader) GetSourceByName 

func (r *SpecReader) GetSourceByName(name string) *Source

func (*SpecReader) GetSourceWarningsByName 

func (r *SpecReader) GetSourceWarningsByName(name string) Warnings

func (*SpecReader) GetTransformerWarningsByName 

func (r *SpecReader) GetTransformerWarningsByName(name string) Warnings

type Transformer 

type Transformer struct {
	Metadata

	// Transformer plugin own (nested) spec
	Spec map[string]any `json:"spec,omitempty"`
}

Transformer plugin spec

func (*Transformer) GetWarnings 

func (*Transformer) GetWarnings() Warnings

func (*Transformer) UnmarshalSpec 

func (d *Transformer) UnmarshalSpec(out any) error

func (*Transformer) Validate 

func (d *Transformer) Validate() error

type Variables 

type Variables struct {
	Plugins map[string]PluginVariables `json:"plugins"`
}

type Warnings 

type Warnings map[string]string

Warnings is a map of field name to string, used mainly for deprecation notices.

type WriteMode 

type WriteMode int
const (
	WriteModeOverwriteDeleteStale WriteMode = iota
	WriteModeOverwrite
	WriteModeAppend
)

func WriteModeFromString 

func WriteModeFromString(s string) (WriteMode, error)

func (WriteMode) JSONSchemaExtend 

func (WriteMode) JSONSchemaExtend(sc *jsonschema.Schema)

func (WriteMode) MarshalJSON 

func (m WriteMode) MarshalJSON() ([]byte, error)

func (WriteMode) String 

func (m WriteMode) String() string

func (*WriteMode) UnmarshalJSON 

func (m *WriteMode) UnmarshalJSON(data []byte) (err error)

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.