s3

package
v0.0.0-rc12 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Dec 2, 2024 License: Apache-2.0 Imports: 29 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	NoEncoding   = Encoding("UNCOMPRESSED")
	GzipEncoding = Encoding("GZIP")
)
View Source 
const (
	ProviderType = abstract.ProviderType("s3")
)
View Source 
const (
	XYaCloudTokenHeader string = "X-YaCloud-SubjectToken"
)

Variables

View Source 
var (
	UnparsedPolicyFail     = UnparsedPolicy("fail")
	UnparsedPolicyContinue = UnparsedPolicy("continue")
	UnparsedPolicyRetry    = UnparsedPolicy("retry")
)

Functions

func CreateBucket 

func CreateBucket(t *testing.T, cfg *S3Source)

func EnvOrDefault 

func EnvOrDefault(key string, def string) string

func NewAWSSession 

func NewAWSSession(lgr log.Logger, bucket string, cfg ConnectionConfig) (*session.Session, error)

func PrepareTestCase 

func PrepareTestCase(t *testing.T, cfg *S3Source, casePath string)

func UploadOne 

func UploadOne(t *testing.T, cfg *S3Source, fname string)

Types

type AdditionalOptions 

type AdditionalOptions struct {
	NullValues             []string `json:"null_values,omitempty"`
	TrueValues             []string `json:"true_values,omitempty"`
	FalseValues            []string `json:"false_values,omitempty"`
	DecimalPoint           string   `json:"decimal_point,omitempty"`
	StringsCanBeNull       bool     `json:"strings_can_be_null,omitempty"`        // default false
	QuotedStringsCanBeNull bool     `json:"quoted_strings_can_be_null,omitempty"` // default true
	IncludeColumns         []string `json:"include_columns,omitempty"`
	IncludeMissingColumns  bool     `json:"include_missing_columns,omitempty"` // default false
	TimestampParsers       []string `json:"timestamp_parsers,omitempty"`
}

type AdvancedOptions 

type AdvancedOptions struct {
	SkipRows                int64    `json:"skip_rows,omitempty"`
	SkipRowsAfterNames      int64    `json:"skip_rows_after_names,omitempty"`
	ColumnNames             []string `json:"column_names,omitempty"`
	AutogenerateColumnNames bool     `json:"autogenerate_column_names,omitempty"` // default true
}

type CSVSetting 

type CSVSetting struct {
	Delimiter               string
	QuoteChar               string
	EscapeChar              string
	Encoding                string
	DoubleQuote             bool
	NewlinesInValue         bool
	BlockSize               int64
	AdditionalReaderOptions AdditionalOptions
	AdvancedOptions         AdvancedOptions
}

type ConnectionConfig 

type ConnectionConfig struct {
	AccessKey        string
	S3ForcePathStyle bool
	SecretKey        model.SecretString
	Endpoint         string
	UseSSL           bool
	VerifySSL        bool
	Region           string
	ServiceAccountID string
}

type Encoding 

type Encoding string

type EventSource 

type EventSource struct {
	SQS    *SQS
	SNS    *SNS
	PubSub *PubSub
}

type Format 

type Format struct {
	CSVSetting     *CSVSetting
	JSONLSetting   *JSONLSetting
	ParquetSetting *ParquetSetting
	ProtoParser    *ProtoSetting
}

type JSONLSetting 

type JSONLSetting struct {
	NewlinesInValue         bool
	BlockSize               int64
	UnexpectedFieldBehavior UnexpectedFieldBehavior
}

type ParquetSetting 

type ParquetSetting struct{}

type ProtoSetting 

type ProtoSetting struct {
	DescFile         []byte
	DescResourceName string
	MessageName      string

	IncludeColumns []protoparser.ColParams
	PrimaryKeys    []string
	PackageType    protoparser.MessagePackageType

	NullKeysAllowed bool
}

type PubSub 

type PubSub struct{} // Will be implemented in ORION-3448

type S3Destination 

type S3Destination struct {
	OutputFormat     dp_model.ParsingFormat
	OutputEncoding   Encoding
	BufferSize       dp_model.BytesSize
	BufferInterval   time.Duration
	Endpoint         string
	Region           string
	AccessKey        string
	S3ForcePathStyle bool
	Secret           string
	ServiceAccountID string
	Layout           string
	LayoutTZ         string
	LayoutColumn     string
	Bucket           string
	UseSSL           bool
	VerifySSL        bool
	PartSize         int64
	Concurrency      int64
	AnyAsString      bool
}

func PrepareS3 

func PrepareS3(t *testing.T, bucket string, format model.ParsingFormat, encoding Encoding) *S3Destination

func (*S3Destination) BuffererConfig 

func (d *S3Destination) BuffererConfig() bufferer.BuffererConfig

func (*S3Destination) CleanupMode 

func (d *S3Destination) CleanupMode() dp_model.CleanupType

func (*S3Destination) Compatible 

func (d *S3Destination) Compatible(src dp_model.Source, _ abstract.TransferType) error

func (*S3Destination) ConnectionConfig 

func (d *S3Destination) ConnectionConfig() ConnectionConfig

func (*S3Destination) GetProviderType 

func (d *S3Destination) GetProviderType() abstract.ProviderType

func (S3Destination) IsDestination 

func (S3Destination) IsDestination()

func (*S3Destination) Transformer 

func (d *S3Destination) Transformer() map[string]string

func (*S3Destination) Validate 

func (d *S3Destination) Validate() error

func (*S3Destination) WithDefaults 

func (d *S3Destination) WithDefaults()

type S3Source 

type S3Source struct {
	Bucket           string
	ConnectionConfig ConnectionConfig
	PathPrefix       string

	HideSystemCols bool // to hide system cols `__file_name` and `__row_index` cols from out struct
	ReadBatchSize  int
	InflightLimit  int64

	// s3 hold always single table, and TableID of such table defined by user
	TableName      string
	TableNamespace string

	InputFormat  model.ParsingFormat
	OutputSchema []abstract.ColSchema

	AirbyteFormat string // this is for backward compatibility with airbyte. we store raw format for later parsing.
	PathPattern   string

	Concurrency int64

	Format         Format
	EventSource    EventSource
	UnparsedPolicy UnparsedPolicy
}

func PrepareCfg 

func PrepareCfg(t *testing.T, bucket string, format model.ParsingFormat) *S3Source

func (*S3Source) GetProviderType 

func (s *S3Source) GetProviderType() abstract.ProviderType

func (*S3Source) IsAbstract2 

func (s *S3Source) IsAbstract2(model.Destination) bool

func (*S3Source) IsAppendOnly 

func (s *S3Source) IsAppendOnly() bool

func (*S3Source) IsSource 

func (s *S3Source) IsSource()

func (*S3Source) TableID 

func (s *S3Source) TableID() abstract.TableID

func (*S3Source) Validate 

func (s *S3Source) Validate() error

func (*S3Source) WithDefaults 

func (s *S3Source) WithDefaults()

type SNS 

type SNS struct{} // Will be implemented in ORION-3447

type SQS 

type SQS struct {
	QueueName        string
	OwnerAccountID   string
	ConnectionConfig ConnectionConfig
}

type UnexpectedFieldBehavior 

type UnexpectedFieldBehavior int
const (
	Unspecified UnexpectedFieldBehavior = iota
	Infer
	Ignore
	Error
)

type UnparsedPolicy 

type UnparsedPolicy string

Source Files

View all Source files

Directories

Path Synopsis
testutil

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.