Documentation ¶
Overview ¶
Package schema defines types supported by tables in source plugins
nolint:gocritic
Index ¶
- Variables
- func TransformWithTransformer(transformer CQTypeTransformer, values CQTypes) []any
- func ValidateTable(t *Table) error
- type ArrayDimension
- type Bool
- type BoolTransformer
- type Bytea
- type ByteaTransformer
- type CIDR
- type CIDRArray
- type CIDRArrayTransformer
- type CIDRTransformer
- type CQType
- type CQTypeTransformer
- type CQTypes
- type ClientMeta
- type Column
- type ColumnCreationOptions
- type ColumnList
- type ColumnResolver
- type DefaultTransformer
- func (*DefaultTransformer) TransformBool(v *Bool) any
- func (*DefaultTransformer) TransformBytea(v *Bytea) any
- func (*DefaultTransformer) TransformCIDR(v *CIDR) any
- func (*DefaultTransformer) TransformCIDRArray(v *CIDRArray) any
- func (*DefaultTransformer) TransformFloat8(v *Float8) any
- func (*DefaultTransformer) TransformInet(v *Inet) any
- func (*DefaultTransformer) TransformInetArray(v *InetArray) any
- func (*DefaultTransformer) TransformInt8(v *Int8) any
- func (*DefaultTransformer) TransformInt8Array(v *Int8Array) any
- func (*DefaultTransformer) TransformJSON(v *JSON) any
- func (*DefaultTransformer) TransformMacaddr(v *Macaddr) any
- func (*DefaultTransformer) TransformMacaddrArray(v *MacaddrArray) any
- func (*DefaultTransformer) TransformText(v *Text) any
- func (*DefaultTransformer) TransformTextArray(v *TextArray) any
- func (*DefaultTransformer) TransformTimestamptz(v *Timestamptz) any
- func (*DefaultTransformer) TransformUUID(v *UUID) any
- func (*DefaultTransformer) TransformUUIDArray(v *UUIDArray) any
- type DestinationResource
- type Float8
- type Float8Transformer
- type Inet
- type InetArray
- type InetArrayTransformer
- type InetTransformer
- type InfinityModifier
- type Int8
- type Int8Array
- type Int8ArrayTransformer
- type Int8Transformer
- type JSON
- type JSONTransformer
- type LengthTableValidator
- type Macaddr
- type MacaddrArray
- type MacaddrArrayTransformer
- type MacaddrTransformer
- type Multiplexer
- type Resource
- func (r *Resource) CalculateCQID(deterministicCQID bool) error
- func (r *Resource) Columns() []string
- func (r *Resource) Get(columnName string) CQType
- func (r *Resource) GetItem() any
- func (r *Resource) GetValues() CQTypes
- func (r *Resource) Set(columnName string, value any) error
- func (r *Resource) SetItem(item any)
- func (r *Resource) ToDestinationResource() DestinationResource
- func (r *Resource) Validate() error
- type Resources
- type RowResolver
- type Status
- type SyncSummary
- type Table
- func (t *Table) Column(name string) *Column
- func (t *Table) Copy(parent *Table) *Table
- func (t *Table) GetChanges(old *Table) []TableColumnChange
- func (t *Table) IncrementalKeys() []string
- func (t *Table) OverwriteOrAddColumn(column *Column)
- func (t *Table) PrimaryKeys() []string
- func (t *Table) TableNames() []string
- func (t *Table) ValidateColumnNames() error
- func (t *Table) ValidateDuplicateColumns() error
- func (t *Table) ValidateName() error
- type TableColumnChange
- type TableColumnChangeType
- type TableResolver
- type TableValidator
- type Tables
- func (tt Tables) FilterDfs(tables, skipTables []string, skipDependentTables bool) (Tables, error)
- func (tt Tables) FilterDfsFunc(include, exclude func(*Table) bool, skipDependentTables bool) Tables
- func (tt Tables) FlattenTables() Tables
- func (tt Tables) Get(name string) *Table
- func (tt Tables) GetTopLevel(name string) *Table
- func (tt Tables) TableNames() []string
- func (tt Tables) ValidateColumnNames() error
- func (tt Tables) ValidateDuplicateColumns() error
- func (tt Tables) ValidateDuplicateTables() error
- func (tt Tables) ValidateTableNames() error
- type Text
- type TextArray
- type TextArrayTransformer
- type TextTransformer
- type Timestamptz
- func (dst *Timestamptz) DecodeText(src []byte) error
- func (dst *Timestamptz) Equal(src CQType) bool
- func (dst Timestamptz) Get() any
- func (dst *Timestamptz) GetStatus() Status
- func (dst *Timestamptz) Set(src any) error
- func (dst *Timestamptz) Size() int
- func (dst *Timestamptz) String() string
- func (*Timestamptz) Type() ValueType
- type TimestamptzTransformer
- type Transform
- type UUID
- type UUIDArray
- type UUIDArrayTransformer
- type UUIDTransformer
- type ValidationError
- type ValueType
Constants ¶
This section is empty.
Variables ¶
var CqIDColumn = Column{ Name: "_cq_id", Type: TypeUUID, Description: "Internal CQ ID of the row", CreationOptions: ColumnCreationOptions{ NotNull: true, Unique: true, }, }
These columns are managed and populated by the source plugins
var CqParentIDColumn = Column{ Name: "_cq_parent_id", Type: TypeUUID, Description: "Internal CQ ID of the parent row", Resolver: parentCqUUIDResolver(), IgnoreInTests: true, }
var CqSourceNameColumn = Column{ Name: "_cq_source_name", Type: TypeString, Description: "Internal CQ row that references the source plugin name data was retrieved", }
var CqSyncTimeColumn = Column{ Name: "_cq_sync_time", Type: TypeTimestamp, Description: "Internal CQ row of when sync was started (this will be the same for all rows in a single fetch)", }
These columns are managed and populated by the destination plugin.
Functions ¶
func TransformWithTransformer ¶ added in v0.13.23
func TransformWithTransformer(transformer CQTypeTransformer, values CQTypes) []any
func ValidateTable ¶
Types ¶
type ArrayDimension ¶ added in v0.13.23
type BoolTransformer ¶ added in v0.13.23
type ByteaTransformer ¶ added in v0.13.23
type CIDRArray ¶ added in v0.13.23
type CIDRArray struct { Elements []CIDR Dimensions []ArrayDimension Status Status }
type CIDRArrayTransformer ¶ added in v0.13.23
type CIDRTransformer ¶ added in v0.13.23
type CQType ¶ added in v0.13.23
type CQType interface { Set(v any) error Get() any String() string // Used only in testing Equal(CQType) bool Type() ValueType Size() int GetStatus() Status }
func NewCqTypeFromValueType ¶ added in v0.13.23
type CQTypeTransformer ¶ added in v0.13.23
type CQTypeTransformer interface { BoolTransformer ByteaTransformer CIDRArrayTransformer CIDRTransformer Float8Transformer InetArrayTransformer InetTransformer Int8ArrayTransformer Int8Transformer JSONTransformer MacaddrArrayTransformer MacaddrTransformer TextArrayTransformer TextTransformer TimestamptzTransformer UUIDArrayTransformer UUIDTransformer }
type CQTypes ¶ added in v0.13.23
type CQTypes []CQType
func (CQTypes) MarshalJSON ¶ added in v0.13.23
func (CQTypes) Size ¶ added in v1.22.0
Size returns total number of bytes occupied by all values this useful to understand how much data is being transferred, rather than just number of resources.
func (*CQTypes) UnmarshalJSON ¶ added in v0.13.23
type ClientMeta ¶
type ClientMeta interface {
ID() string
}
type Column ¶
type Column struct { // Name of column Name string `json:"name,omitempty"` // Value Type of column i.e String, UUID etc' Type ValueType `json:"type,omitempty"` // Description about column, this description is added as a comment in the database Description string `json:"-"` // Column Resolver allows to set your own data for a column; this can be an API call, setting multiple embedded values, etc Resolver ColumnResolver `json:"-"` // Creation options allow modifying how column is defined when table is created CreationOptions ColumnCreationOptions `json:"creation_options,omitempty"` // IgnoreInTests is used to skip verifying the column is non-nil in integration tests. // By default, integration tests perform a fetch for all resources in cloudquery's test account, and // verify all columns are non-nil. // If IgnoreInTests is true, verification is skipped for this column. // Used when it is hard to create a reproducible environment with this column being non-nil (e.g. various error columns). IgnoreInTests bool `json:"-"` }
Column definition for Table
type ColumnCreationOptions ¶
type ColumnCreationOptions struct { PrimaryKey bool `json:"primary_key,omitempty"` NotNull bool `json:"not_null,omitempty"` // IncrementalKey is a flag that indicates if the column is used as part of an incremental key. // It is mainly used for documentation purposes, but may also be used as part of ensuring that // migrations are done correctly. IncrementalKey bool `json:"incremental_key"` Unique bool `json:"unique,omitempty"` }
ColumnCreationOptions allow modification of how column is defined when table is created
type ColumnList ¶
type ColumnList []Column
func (ColumnList) Get ¶
func (c ColumnList) Get(name string) *Column
func (ColumnList) Index ¶ added in v0.13.16
func (c ColumnList) Index(col string) int
func (ColumnList) Names ¶
func (c ColumnList) Names() []string
func (ColumnList) String ¶ added in v1.36.1
func (c ColumnList) String() string
func (*ColumnList) UnmarshalJSON ¶ added in v0.13.23
func (c *ColumnList) UnmarshalJSON(data []byte) (err error)
type ColumnResolver ¶
ColumnResolver is called for each row received in TableResolver's data fetch. execution holds all relevant information regarding execution as well as the Column called. resource holds the current row we are resolving the column for.
func ParentColumnResolver ¶ added in v0.11.0
func ParentColumnResolver(name string) ColumnResolver
ParentColumnResolver resolves a column from the parent's table data, if name isn't set the column will be set to null
func PathResolver ¶
func PathResolver(path string) ColumnResolver
PathResolver resolves a field in the Resource.Item
Examples: PathResolver("Field") PathResolver("InnerStruct.Field") PathResolver("InnerStruct.InnerInnerStruct.Field")
type DefaultTransformer ¶ added in v0.13.23
type DefaultTransformer struct { }
func (*DefaultTransformer) TransformBool ¶ added in v0.13.23
func (*DefaultTransformer) TransformBool(v *Bool) any
func (*DefaultTransformer) TransformBytea ¶ added in v0.13.23
func (*DefaultTransformer) TransformBytea(v *Bytea) any
func (*DefaultTransformer) TransformCIDR ¶ added in v0.13.23
func (*DefaultTransformer) TransformCIDR(v *CIDR) any
func (*DefaultTransformer) TransformCIDRArray ¶ added in v0.13.23
func (*DefaultTransformer) TransformCIDRArray(v *CIDRArray) any
func (*DefaultTransformer) TransformFloat8 ¶ added in v0.13.23
func (*DefaultTransformer) TransformFloat8(v *Float8) any
func (*DefaultTransformer) TransformInet ¶ added in v0.13.23
func (*DefaultTransformer) TransformInet(v *Inet) any
func (*DefaultTransformer) TransformInetArray ¶ added in v0.13.23
func (*DefaultTransformer) TransformInetArray(v *InetArray) any
func (*DefaultTransformer) TransformInt8 ¶ added in v0.13.23
func (*DefaultTransformer) TransformInt8(v *Int8) any
func (*DefaultTransformer) TransformInt8Array ¶ added in v0.13.23
func (*DefaultTransformer) TransformInt8Array(v *Int8Array) any
func (*DefaultTransformer) TransformJSON ¶ added in v0.13.23
func (*DefaultTransformer) TransformJSON(v *JSON) any
func (*DefaultTransformer) TransformMacaddr ¶ added in v0.13.23
func (*DefaultTransformer) TransformMacaddr(v *Macaddr) any
func (*DefaultTransformer) TransformMacaddrArray ¶ added in v0.13.23
func (*DefaultTransformer) TransformMacaddrArray(v *MacaddrArray) any
func (*DefaultTransformer) TransformText ¶ added in v0.13.23
func (*DefaultTransformer) TransformText(v *Text) any
func (*DefaultTransformer) TransformTextArray ¶ added in v0.13.23
func (*DefaultTransformer) TransformTextArray(v *TextArray) any
func (*DefaultTransformer) TransformTimestamptz ¶ added in v0.13.23
func (*DefaultTransformer) TransformTimestamptz(v *Timestamptz) any
func (*DefaultTransformer) TransformUUID ¶ added in v0.13.23
func (*DefaultTransformer) TransformUUID(v *UUID) any
func (*DefaultTransformer) TransformUUIDArray ¶ added in v0.13.23
func (*DefaultTransformer) TransformUUIDArray(v *UUIDArray) any
type DestinationResource ¶ added in v0.13.16
This struct is what we send over the wire to destination. We dont want to reuse the same struct as otherwise we will have to comment on fields which don't get sent over the wire but still accessible code wise
type Float8Transformer ¶ added in v0.13.23
type Inet ¶ added in v0.13.23
Inet represents both inet and cidr PostgreSQL types.
func (*Inet) UnmarshalJSON ¶ added in v1.5.3
workaround this Golang bug: https://github.com/golang/go/issues/35727
type InetArray ¶ added in v0.13.23
type InetArray struct { Elements []Inet Dimensions []ArrayDimension Status Status }
type InetArrayTransformer ¶ added in v0.13.23
type InetTransformer ¶ added in v0.13.23
type InfinityModifier ¶ added in v0.13.23
type InfinityModifier int8
const ( Infinity InfinityModifier = 1 None InfinityModifier = 0 NegativeInfinity InfinityModifier = -Infinity )
type Int8Array ¶ added in v0.13.23
type Int8Array struct { Elements []Int8 Dimensions []ArrayDimension Status Status }
type Int8ArrayTransformer ¶ added in v0.13.23
type Int8Transformer ¶ added in v0.13.23
type JSONTransformer ¶ added in v0.13.23
type LengthTableValidator ¶
type LengthTableValidator struct{}
func (LengthTableValidator) Validate ¶
func (LengthTableValidator) Validate(t *Table) error
type Macaddr ¶ added in v0.13.23
type Macaddr struct { Addr net.HardwareAddr Status Status }
type MacaddrArray ¶ added in v0.13.23
type MacaddrArray struct { Elements []Macaddr Dimensions []ArrayDimension Status Status }
func (*MacaddrArray) Equal ¶ added in v0.13.23
func (dst *MacaddrArray) Equal(src CQType) bool
func (MacaddrArray) Get ¶ added in v0.13.23
func (dst MacaddrArray) Get() any
func (*MacaddrArray) GetStatus ¶ added in v1.17.0
func (dst *MacaddrArray) GetStatus() Status
func (*MacaddrArray) Set ¶ added in v0.13.23
func (dst *MacaddrArray) Set(src any) error
func (*MacaddrArray) Size ¶ added in v1.22.0
func (dst *MacaddrArray) Size() int
func (*MacaddrArray) String ¶ added in v0.13.23
func (dst *MacaddrArray) String() string
func (*MacaddrArray) Type ¶ added in v0.13.23
func (*MacaddrArray) Type() ValueType
type MacaddrArrayTransformer ¶ added in v0.13.23
type MacaddrArrayTransformer interface {
TransformMacaddrArray(*MacaddrArray) any
}
type MacaddrTransformer ¶ added in v0.13.23
type Multiplexer ¶ added in v0.7.1
type Multiplexer func(meta ClientMeta) []ClientMeta
type Resource ¶
type Resource struct { // Original resource item that wa from prior resolve Item any // Set if this is an embedded table Parent *Resource // internal fields Table *Table // contains filtered or unexported fields }
Resource represents a row in it's associated table, it carries a reference to the original item, and automatically generates an Id based on Table's Columns. Resource data can be accessed by the Get and Set methods
func (*Resource) CalculateCQID ¶ added in v1.41.0
func (*Resource) Set ¶
Set sets a column with value. This does validation and conversion to one of concrete it returns an error just for backward compatibility and panics in case it fails
func (*Resource) SetItem ¶ added in v0.0.11
Override original item (this is useful for apis that follow list/details pattern)
func (*Resource) ToDestinationResource ¶ added in v0.13.16
func (r *Resource) ToDestinationResource() DestinationResource
type RowResolver ¶
type RowResolver func(ctx context.Context, meta ClientMeta, resource *Resource) error
type SyncSummary ¶ added in v0.12.0
This is deprecated
type Table ¶
type Table struct { // Name of table Name string `json:"name"` // Title to be used in documentation (optional: will be generated from name if not set) Title string `json:"title"` // table description Description string `json:"description"` // Columns are the set of fields that are part of this table Columns ColumnList `json:"columns"` // Relations are a set of related tables defines Relations Tables `json:"relations"` // Transform Transform Transform `json:"-"` // Resolver is the main entry point to fetching table data and Resolver TableResolver `json:"-"` // Multiplex returns re-purposed meta clients. The sdk will execute the table with each of them Multiplex Multiplexer `json:"-"` // PostResourceResolver is called after all columns have been resolved, but before the Resource is sent to be inserted. The ordering of resolvers is: // (Table) Resolver → PreResourceResolver → ColumnResolvers → PostResourceResolver PostResourceResolver RowResolver `json:"-"` // PreResourceResolver is called before all columns are resolved but after Resource is created. The ordering of resolvers is: // (Table) Resolver → PreResourceResolver → ColumnResolvers → PostResourceResolver PreResourceResolver RowResolver `json:"-"` // IsIncremental is a flag that indicates if the table is incremental or not. This flag mainly affects how the table is // documented. IsIncremental bool // IgnoreInTests is used to exclude a table from integration tests. // By default, integration tests fetch all resources from cloudquery's test account, and verify all tables // have at least one row. // When IgnoreInTests is true, integration tests won't fetch from this table. // Used when it is hard to create a reproducible environment with a row in this table. IgnoreInTests bool `json:"ignore_in_tests"` // Parent is the parent table in case this table is called via parent table (i.e. relation) Parent *Table `json:"-"` PkConstraintName string `json:"pk_constraint_name"` }
func (*Table) GetChanges ¶ added in v1.38.0
func (t *Table) GetChanges(old *Table) []TableColumnChange
Get Changes returns changes between two tables when t is the new one and old is the old one.
func (*Table) IncrementalKeys ¶ added in v1.24.0
func (*Table) OverwriteOrAddColumn ¶ added in v0.13.0
If the column with the same name exists, overwrites it. Otherwise, adds the column to the beginning of the table.
func (*Table) PrimaryKeys ¶ added in v0.0.4
func (*Table) TableNames ¶
func (*Table) ValidateColumnNames ¶ added in v0.13.11
func (*Table) ValidateDuplicateColumns ¶ added in v0.0.4
func (*Table) ValidateName ¶ added in v0.13.11
type TableColumnChange ¶ added in v1.38.0
type TableColumnChange struct { Type TableColumnChangeType ColumnName string Current Column Previous Column }
func (TableColumnChange) String ¶ added in v1.38.0
func (t TableColumnChange) String() string
type TableColumnChangeType ¶ added in v1.38.0
type TableColumnChangeType int
const ( TableColumnChangeTypeUnknown TableColumnChangeType = iota TableColumnChangeTypeAdd TableColumnChangeTypeUpdate TableColumnChangeTypeRemove )
func (TableColumnChangeType) String ¶ added in v1.38.0
func (t TableColumnChangeType) String() string
type TableResolver ¶
type TableResolver func(ctx context.Context, meta ClientMeta, parent *Resource, res chan<- any) error
TableResolver is the main entry point when a table is sync is called.
Table resolver has 3 main arguments: - meta(ClientMeta): is the client returned by the plugin.Provider Configure call - parent(Resource): resource is the parent resource in case this table is called via parent table (i.e. relation) - res(chan any): is a channel to pass results fetched by the TableResolver
type TableValidator ¶
type Tables ¶ added in v0.0.4
type Tables []*Table
func (Tables) FilterDfsFunc ¶ added in v1.24.2
func (Tables) FlattenTables ¶ added in v1.8.1
func (Tables) Get ¶ added in v0.13.15
Get returns a table by name. Returns top-level tables and relations.
func (Tables) GetTopLevel ¶ added in v1.5.1
GetTopLevel returns a table by name. Only returns the table if it is in top-level list.
func (Tables) TableNames ¶ added in v0.0.4
func (Tables) ValidateColumnNames ¶ added in v0.13.11
func (Tables) ValidateDuplicateColumns ¶ added in v0.0.4
func (Tables) ValidateDuplicateTables ¶ added in v0.0.6
func (Tables) ValidateTableNames ¶ added in v0.13.11
type Text ¶ added in v0.13.23
type TextArray ¶ added in v0.13.23
type TextArray struct { Elements []Text Dimensions []ArrayDimension Status Status }
type TextArrayTransformer ¶ added in v0.13.23
type TextTransformer ¶ added in v0.13.23
type Timestamptz ¶ added in v0.13.23
type Timestamptz struct { Time time.Time Status Status InfinityModifier InfinityModifier }
func (*Timestamptz) DecodeText ¶ added in v0.13.23
func (dst *Timestamptz) DecodeText(src []byte) error
func (*Timestamptz) Equal ¶ added in v0.13.23
func (dst *Timestamptz) Equal(src CQType) bool
func (Timestamptz) Get ¶ added in v0.13.23
func (dst Timestamptz) Get() any
func (*Timestamptz) GetStatus ¶ added in v1.17.0
func (dst *Timestamptz) GetStatus() Status
func (*Timestamptz) Set ¶ added in v0.13.23
func (dst *Timestamptz) Set(src any) error
func (*Timestamptz) Size ¶ added in v1.22.0
func (dst *Timestamptz) Size() int
func (*Timestamptz) String ¶ added in v0.13.23
func (dst *Timestamptz) String() string
func (*Timestamptz) Type ¶ added in v0.13.23
func (*Timestamptz) Type() ValueType
type TimestamptzTransformer ¶ added in v0.13.23
type TimestamptzTransformer interface {
TransformTimestamptz(*Timestamptz) any
}
type UUIDArray ¶ added in v0.13.23
type UUIDArray struct { Elements []UUID Dimensions []ArrayDimension Status Status }
type UUIDArrayTransformer ¶ added in v0.13.23
type UUIDTransformer ¶ added in v0.13.23
type ValidationError ¶ added in v1.24.1
func (*ValidationError) Error ¶ added in v1.24.1
func (e *ValidationError) Error() string
func (*ValidationError) MaskedError ¶ added in v1.24.1
func (e *ValidationError) MaskedError() string
this prints the error without the value
func (*ValidationError) Unwrap ¶ added in v1.24.1
func (e *ValidationError) Unwrap() error
Source Files ¶
- array.go
- bool.go
- bytea.go
- cidr.go
- cidr_array.go
- column.go
- convert.go
- doc.go
- errors.go
- float8.go
- inet.go
- inet_array.go
- int8.go
- int8_array.go
- json.go
- macaddr.go
- macaddr_array.go
- meta.go
- resolvers.go
- resource.go
- table.go
- text.go
- text_array.go
- timestamptz.go
- transformer.go
- types.go
- uuid.go
- uuid_array.go
- validators.go