types

package
v0.1.1 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Feb 10, 2025 License: Apache-2.0 Imports: 12 Imported by: 1

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

func ConfigDataFromProto 

func ConfigDataFromProto[T ConfigData](data *proto.ConfigData) (T, error)

Types

type ArtifactInfo 

type ArtifactInfo struct {
	// this is the original name of the artifact
	Name string `json:"original_name"`

	// enrichment values passed from the source to the collection to include in the enrichment process
	SourceEnrichment *schema.SourceEnrichment `json:"-"`
	Timestamp        time.Time                `json:"timestamp"`
}

func ArtifactInfoFromProto 

func ArtifactInfoFromProto(info *proto.ArtifactInfo) *ArtifactInfo

func NewArtifactInfo 

func NewArtifactInfo(path string, sourceEnrichment *schema.SourceEnrichment, granularity time.Duration) (*ArtifactInfo, error)

func (*ArtifactInfo) GetTimestamp 

func (a *ArtifactInfo) GetTimestamp() time.Time

func (*ArtifactInfo) Identifier 

func (a *ArtifactInfo) Identifier() string

func (*ArtifactInfo) ToProto 

func (a *ArtifactInfo) ToProto() *proto.ArtifactInfo

type CollectRequest 

type CollectRequest struct {
	TableName     string
	PartitionName string

	// unique identifier for collection execution this will be used as base for the filename fo the resultiung JSONL files
	ExecutionId string
	// the parent folder for all collection related files (JSONL files, temp source files)
	CollectionTempDir string
	// the filepath for the collection state json file
	CollectionStatePath string
	// the source to use (with raw config)
	SourceData *SourceConfigData
	// the source format to use (with raw config)
	SourceFormat *FormatConfigData
	// the raw hcl of the connection
	ConnectionData *ConnectionConfigData
	// the collection start time
	From time.Time
	// the custom table definition, if specified
	CustomTable *Table
}

CollectRequest is an sdk type which is mapped from the proto.CollectRequest

func CollectRequestFromProto 

func CollectRequestFromProto(pr *proto.CollectRequest) (*CollectRequest, error)

type Config 

type Config interface {
	Validate() error
	Identifier() string
}

Config is an interface that all configuration structs must implement - this includes: - table config - source config - connection config

type ConfigData 

type ConfigData interface {
	GetHcl() []byte
	GetRange() hcl.Range
	Identifier() string
	AsProto() *proto.ConfigData
	GetConfigType() string
}

ConfigData is a struct used to contain the config data used to configure a Collection or Source it contains the type of cource/collection, as well as the raw HCL config which the newly instantiated object must parse into the appropriate type

type ConfigDataImpl 

type ConfigDataImpl struct {
	Hcl   []byte
	Range hcl.Range
	// Id represent the type of the config:
	// - if this is a partition config, this will be the table name
	// - if this is a source config, this will be the source type
	// - if this is a connection config, this will be the connection type (i.e. plugin name)
	Id string
	// ConfigType is the type of the config data, i.e. connection, source, partition
	ConfigType string
}

func (*ConfigDataImpl) AsProto 

func (c *ConfigDataImpl) AsProto() *proto.ConfigData

func (*ConfigDataImpl) GetConfigType 

func (c *ConfigDataImpl) GetConfigType() string

GetConfigType returns the type of the config data

func (*ConfigDataImpl) GetHcl 

func (c *ConfigDataImpl) GetHcl() []byte

GetHcl returns the HCL config data

func (*ConfigDataImpl) GetRange 

func (c *ConfigDataImpl) GetRange() hcl.Range

GetRange returns the HCL range of the config data

func (*ConfigDataImpl) Identifier 

func (c *ConfigDataImpl) Identifier() string

Identifier returns the identifier of the config data

type ConnectionConfigData 

type ConnectionConfigData struct {
	*ConfigDataImpl
	Type string
}

func NewConnectionConfigData 

func NewConnectionConfigData(hcl []byte, decRange hcl.Range, ty string) *ConnectionConfigData

type DownloadedArtifactInfo 

type DownloadedArtifactInfo struct {
	ArtifactInfo
	// the path of the downloaded artifact
	LocalName string `json:"local_name"`

	Size int64 `json:"size"`
}

DownloadedArtifactInfo contains information about a downloaded artifact is the same as ArtifactInfo, but with a size field

func DownloadedArtifactInfoFromProto 

func DownloadedArtifactInfoFromProto(info *proto.DownloadedArtifactInfo) *DownloadedArtifactInfo

func NewDownloadedArtifactInfo 

func NewDownloadedArtifactInfo(artifactInfo *ArtifactInfo, localName string, size int64) *DownloadedArtifactInfo

func (*DownloadedArtifactInfo) ToProto 

func (a *DownloadedArtifactInfo) ToProto() *proto.DownloadedArtifactInfo

type DynamicTableConfig 

type DynamicTableConfig interface {
	GetSchema() *schema.RowSchema
}

DynamicTableConfig is an interface that all dynamic table configuration structs must implement

type ExtensionLookup 

type ExtensionLookup map[string]struct{}

func NewExtensionLookup 

func NewExtensionLookup(extensions []string) ExtensionLookup

func (ExtensionLookup) IsValid 

func (l ExtensionLookup) IsValid(path string) bool

type FormatConfigData 

type FormatConfigData struct {
	*ConfigDataImpl
	Type string
}

func NewFormatConfigData 

func NewFormatConfigData(hcl []byte, decRange hcl.Range, formatType string) *FormatConfigData

type JSONString 

type JSONString string

func (*JSONString) UnmarshalJSON 

func (s *JSONString) UnmarshalJSON(data []byte) error

type RowData 

type RowData struct {
	Data             any
	SourceEnrichment *schema.SourceEnrichment
}

RowData is a container for the data and metadata of an row It is used to pass data the [Loader] The RowData returned by the loader is used as the payload of a [events.Row] which is sent to the [table.Table]

type RowStruct 

type RowStruct interface {
	Validate() error
	GetCommonFields() schema.CommonFields
}

type SourceConfigData 

type SourceConfigData struct {
	*ConfigDataImpl
	Type           string
	ReattachConfig *SourcePluginReattach
}

func NewSourceConfigData 

func NewSourceConfigData(hcl []byte, decRange hcl.Range, sourceType string) *SourceConfigData

func (*SourceConfigData) SetReattach 

func (d *SourceConfigData) SetReattach(pr *proto.SourcePluginReattach)

type SourcePluginReattach 

type SourcePluginReattach struct {
	Reattach *plugin.ReattachConfig

	Plugin     string
	SourceType string
}

func ReattachFromProto 

func ReattachFromProto(r *proto.SourcePluginReattach) *SourcePluginReattach

type Table 

type Table struct {
	Name   string
	Schema *schema.RowSchema
}

func TableFromProto 

func TableFromProto(pt *proto.Table) *Table

type TimeMillis 

type TimeMillis int64

func (*TimeMillis) UnmarshalJSON 

func (t *TimeMillis) UnmarshalJSON(b []byte) error

type UnixMillis 

type UnixMillis int64

func (*UnixMillis) UnmarshalJSON 

func (u *UnixMillis) UnmarshalJSON(data []byte) error

UnmarshalJSON converts an ISO 8601 formatted time string to Unix milliseconds.

type UpdateCollectionStateRequest 

type UpdateCollectionStateRequest struct {
	// the folder containing collection state files (e.g. last collection time)
	CollectionStatePath string
	// the source to use (with raw config)
	SourceData *SourceConfigData
	// the start time - this will be the new state end time
	From time.Time
}

UpdateCollectionStateRequest is an sdk type which is mapped from the proto.UpdateCollectionStateRequest

func UpdateCollectionStateRequestFromProto 

func UpdateCollectionStateRequestFromProto(pr *proto.UpdateCollectionStateRequest) (*UpdateCollectionStateRequest, error)

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.