schema

Documentation

Overview

Package schema defines types supported by tables in source plugins

Index

• Constants
• Variables
• func DeleteParentIDFilter(id string) func(meta ClientMeta, parent *Resource) []interface{}
• func ParentIDResolver(_ context.Context, _ ClientMeta, r *Resource, c Column) error
• func ValidateTable(t *Table) error
• type ClientMeta
• type Column
  • func SetColumnMeta(c Column, m *ColumnMeta) Column
  • func (c Column) Internal() bool
  • func (c Column) Meta() *ColumnMeta
  • func (c Column) ValidateType(v interface{}) error
• type ColumnCreationOptions
• type ColumnList
  • func (c ColumnList) Get(name string) *Column
  • func (c ColumnList) Names() []string
• type ColumnMeta
• type ColumnResolver
  • func CQUUIDResolver() ColumnResolver
  • func ParentPathResolver(path string) ColumnResolver
  • func ParentResourceFieldResolver(name string) ColumnResolver
  • func PathResolver(path string) ColumnResolver
• type LengthTableValidator
  • func (LengthTableValidator) Validate(t *Table) error
• type Meta
• type Multiplexer
• type ResolverMeta
• type Resource
  • func NewResourceData(t *Table, parent *Resource, fetchTime time.Time, item interface{}) *Resource
  • func (r *Resource) Columns() []string
  • func (r *Resource) Get(key string) interface{}
  • func (r *Resource) ID() uuid.UUID
  • func (r *Resource) PrimaryKeyValue() string
  • func (r *Resource) Set(key string, value interface{}) error
  • func (r *Resource) SetItem(item interface{})
• type Resources
  • func (rr Resources) ColumnNames() []string
  • func (rr Resources) GetIds() []uuid.UUID
  • func (rr Resources) TableName() string
• type RowResolver
• type Table
  • func (t Table) Column(name string) *Column
  • func (t Table) ColumnIndex(name string) int
  • func (t Table) PrimaryKeys() []string
  • func (t Table) Resolve(ctx context.Context, meta ClientMeta, syncTime time.Time, parent *Resource, ...) int
  • func (t Table) TableNames() []string
  • func (t Table) ValidateDuplicateColumns() error
• type TableResolver
• type TableValidator
• type Tables
  • func (tt Tables) TableNames() []string
  • func (tt Tables) ValidateDuplicateColumns() error
  • func (tt Tables) ValidateDuplicateTables() error
• type ValueType
  • func ValueTypeFromString(s string) ValueType
  • func (v ValueType) String() string

Constants

const FetchIDMetaKey = "cq_fetch_id"

Variables

var CqColumns = []Column{
	CqIDColumn,
	CqFetchTime,
}

var CqFetchTime = Column{Name: "_cq_fetch_time", Type: TypeTimestamp, Description: "Internal CQ row of when fetch was started (this will be the same for all rows in a single fetch)"}

var CqIDColumn = Column{Name: "_cq_id", Type: TypeUUID, Description: "Internal CQ ID of the row", Resolver: CQUUIDResolver()}

Functions

func DeleteParentIDFilter

func DeleteParentIDFilter(id string) func(meta ClientMeta, parent *Resource) []interface{}

DeleteParentIDFilter is mostly used for table relations to delete table data based on parent's cq_id

func ParentIDResolver

func ParentIDResolver(_ context.Context, _ ClientMeta, r *Resource, c Column) error

ParentIDResolver resolves the cq_id from the parent if you want to reference the parent's primary keys use ParentResourceFieldResolver as required.

func ValidateTable

func ValidateTable(t *Table) error

Types

type ClientMeta

type ClientMeta interface {
	Logger() *zerolog.Logger
}

type Column

type Column struct {
	// Name of column
	Name string `json:"name,omitempty"`
	// Value Type of column i.e String, UUID etc'
	Type ValueType `json:"type,omitempty"`
	// Description about column, this description is added as a comment in the database
	Description string `json:"description,omitempty"`
	// Column Resolver allows to set you own data based on resolving this can be an API call or setting multiple embedded values etc'
	Resolver ColumnResolver `json:"-"`
	// Creation options allow modifying how column is defined when table is created
	CreationOptions ColumnCreationOptions `json:"creation_options,omitempty"`
	// IgnoreInTests is used to skip verifying the column is non-nil in integration tests.
	// By default, integration tests perform a fetch for all resources in cloudquery's test account, and
	// verify all columns are non-nil.
	// If IgnoreInTests is true, verification is skipped for this column.
	// Used when it is hard to create a reproducible environment with this column being non-nil (e.g. various error columns).
	IgnoreInTests bool `json:"-"`
	// contains filtered or unexported fields
}

Column definition for Table

func SetColumnMeta

func SetColumnMeta(c Column, m *ColumnMeta) Column

func (Column) Internal

func (c Column) Internal() bool

func (Column) Meta

func (c Column) Meta() *ColumnMeta

func (Column) ValidateType

func (c Column) ValidateType(v interface{}) error

type ColumnCreationOptions

type ColumnCreationOptions struct {
	PrimaryKey bool `json:"primary_key,omitempty"`
}

ColumnCreationOptions allow modification of how column is defined when table is created

type ColumnList

type ColumnList []Column

func (ColumnList) Get

func (c ColumnList) Get(name string) *Column

func (ColumnList) Names

func (c ColumnList) Names() []string

type ColumnMeta

type ColumnMeta struct {
	Resolver     *ResolverMeta
	IgnoreExists bool
}

type ColumnResolver

type ColumnResolver func(ctx context.Context, meta ClientMeta, resource *Resource, c Column) error

ColumnResolver is called for each row received in TableResolver's data fetch. execution holds all relevant information regarding execution as well as the Column called. resource holds the current row we are resolving the column for.

func CQUUIDResolver

func CQUUIDResolver() ColumnResolver

func ParentPathResolver

func ParentPathResolver(path string) ColumnResolver

ParentPathResolver resolves a field from the parent

func ParentResourceFieldResolver

func ParentResourceFieldResolver(name string) ColumnResolver

ParentResourceFieldResolver resolves a field from the parent's resource, the value is expected to be set if name isn't set the field will be set to null

func PathResolver

func PathResolver(path string) ColumnResolver

PathResolver resolves a field in the Resource.Item

Examples: PathResolver("Field") PathResolver("InnerStruct.Field") PathResolver("InnerStruct.InnerInnerStruct.Field")

type LengthTableValidator

type LengthTableValidator struct{}

func (LengthTableValidator) Validate

func (LengthTableValidator) Validate(t *Table) error

type Meta

type Meta struct {
	LastUpdate time.Time `json:"last_updated"`
	FetchID    string    `json:"fetch_id,omitempty"`
}

type Multiplexer

type Multiplexer func(meta ClientMeta) []ClientMeta

type ResolverMeta

type ResolverMeta struct {
	Name    string
	Builtin bool
}

type Resource

type Resource struct {
	// Original resource item that wa from prior resolve
	Item interface{} `json:"-"`
	// Set if this is an embedded table
	Parent *Resource `json:"-"`
	// internal fields
	Table *Table `json:"-"`
	// This is sorted result data by column name
	Data      map[string]interface{} `json:"data"`
	TableName string                 `json:"table_name"`
}

Resource represents a row in it's associated table, it carries a reference to the original item, and automatically generates an Id based on Table's Columns. Resource data can be accessed by the Get and Set methods

func NewResourceData

func NewResourceData(t *Table, parent *Resource, fetchTime time.Time, item interface{}) *Resource

func (*Resource) Columns

func (r *Resource) Columns() []string

func (*Resource) Get

func (r *Resource) Get(key string) interface{}

func (*Resource) ID

func (r *Resource) ID() uuid.UUID

func (*Resource) PrimaryKeyValue

func (r *Resource) PrimaryKeyValue() string

func (*Resource) Set

func (r *Resource) Set(key string, value interface{}) error

func (*Resource) SetItem

func (r *Resource) SetItem(item interface{})

Override original item (this is useful for apis that follow list/details pattern)

type Resources

type Resources []*Resource

func (Resources) ColumnNames

func (rr Resources) ColumnNames() []string

func (Resources) GetIds

func (rr Resources) GetIds() []uuid.UUID

func (Resources) TableName

func (rr Resources) TableName() string

type RowResolver

type RowResolver func(ctx context.Context, meta ClientMeta, resource *Resource) error

type Table

type Table struct {
	// Name of table
	Name string `json:"name"`
	// table description
	Description string `json:"description"`
	// Columns are the set of fields that are part of this table
	Columns ColumnList `json:"columns"`
	// Relations are a set of related tables defines
	Relations Tables `json:"relations"`
	// Resolver is the main entry point to fetching table data and
	Resolver TableResolver `json:"-"`
	// Multiplex returns re-purposed meta clients. The sdk will execute the table with each of them
	Multiplex Multiplexer `json:"-"`
	// PostResourceResolver is called after all columns have been resolved, but before the Resource is sent to be inserted. The ordering of resolvers is:
	//  (Table) Resolver → PreResourceResolver → ColumnResolvers → PostResourceResolver
	PostResourceResolver RowResolver `json:"-"`
	// PreResourceResolver is called before all columns are resolved but after Resource is created. The ordering of resolvers is:
	//  (Table) Resolver → PreResourceResolver → ColumnResolvers → PostResourceResolver
	PreResourceResolver RowResolver `json:"-"`

	// IgnoreInTests is used to exclude a table from integration tests.
	// By default, integration tests fetch all resources from cloudquery's test account, and verify all tables
	// have at least one row.
	// When IgnoreInTests is true, integration tests won't fetch from this table.
	// Used when it is hard to create a reproducible environment with a row in this table.
	IgnoreInTests bool `json:"ignore_in_tests"`

	// Parent is the parent table in case this table is called via parent table (i.e. relation)
	Parent *Table `json:"-"`

	// Serial is used to force a signature change, which forces new table creation and cascading removal of old table and relations
	Serial string `json:"-"`
	// contains filtered or unexported fields
}

func (Table) Column

func (t Table) Column(name string) *Column

func (Table) ColumnIndex

func (t Table) ColumnIndex(name string) int

func (Table) PrimaryKeys

func (t Table) PrimaryKeys() []string

func (Table) Resolve

func (t Table) Resolve(ctx context.Context, meta ClientMeta, syncTime time.Time, parent *Resource, resolvedResources chan<- *Resource) int

Call the table resolver with with all of it's relation for every reolved resource

func (Table) TableNames

func (t Table) TableNames() []string

func (Table) ValidateDuplicateColumns

func (t Table) ValidateDuplicateColumns() error

type TableResolver

type TableResolver func(ctx context.Context, meta ClientMeta, parent *Resource, res chan<- interface{}) error

TableResolver is the main entry point when a table is sync is called.

Table resolver has 3 main arguments: - meta(ClientMeta): is the client returned by the plugin.Provider Configure call - parent(Resource): resource is the parent resource in case this table is called via parent table (i.e. relation) - res(chan interface{}): is a channel to pass results fetched by the TableResolver

type TableValidator

type TableValidator interface {
	Validate(t *Table) error
}

type Tables

type Tables []*Table

func (Tables) TableNames

func (tt Tables) TableNames() []string

func (Tables) ValidateDuplicateColumns

func (tt Tables) ValidateDuplicateColumns() error

func (Tables) ValidateDuplicateTables

func (tt Tables) ValidateDuplicateTables() error

type ValueType

type ValueType int

const (
	TypeInvalid ValueType = iota
	TypeBool
	TypeInt
	TypeFloat
	TypeUUID
	TypeString
	TypeByteArray
	TypeStringArray
	TypeIntArray
	TypeTimestamp
	TypeJSON
	TypeUUIDArray
	TypeInet
	TypeInetArray
	TypeCIDR
	TypeCIDRArray
	TypeMacAddr
	TypeMacAddrArray
)

func ValueTypeFromString

func ValueTypeFromString(s string) ValueType

ValueTypeFromString this function is mainly used by https://github.com/cloudquery/cq-gen

func (ValueType) String

func (v ValueType) String() string