s3

package
v1.6.1-rc1 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jan 12, 2025 License: Apache-2.0 Imports: 24 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	BadRequestErrorCode = "S3BadRequest"
)
View Source 
const DownloadComponent = "S3Downloader"
View Source 
const InputSourceComponent = "S3InputSource"
View Source 
const PublisherComponent = "S3Publisher"
View Source 
const ResultSignerComponent = "S3ResultSigner"

Variables

This section is empty.

Functions

func CanRunS3Test added in v1.0.1 

func CanRunS3Test() bool

func DefaultAWSConfig 

func DefaultAWSConfig() (aws.Config, error)

DefaultAWSConfig returns an AWS configuration with IMDS disabled if it's determined to be unavailable or inaccessible. This prevents delays from failed IMDS calls during credential retrieval.

func HasValidCredentials 

func HasValidCredentials(config aws.Config) bool

HasValidCredentials returns true if the AWS config has valid credentials.

func IsAWSEndpoint added in v1.2.0 

func IsAWSEndpoint(endpoint string) bool

IsAWSEndpoint checks if the given S3 endpoint URL is an AWS endpoint by its suffix. If the endpoint is empty, it is considered an AWS endpoint.

func NewPublisherSpec added in v1.4.0 

func NewPublisherSpec(bucket string, key string, opts ...PublisherOption) (*models.SpecConfig, error)

func NewS3DownloaderError added in v1.5.0 

func NewS3DownloaderError(code bacerrors.ErrorCode, message string) bacerrors.Error

func NewS3InputSourceError added in v1.5.0 

func NewS3InputSourceError(code bacerrors.ErrorCode, format string, a ...any) bacerrors.Error

func NewS3InputSourceServiceError added in v1.5.0 

func NewS3InputSourceServiceError(err error) bacerrors.Error

func NewS3PublisherError added in v1.5.0 

func NewS3PublisherError(code bacerrors.ErrorCode, message string) bacerrors.Error

func NewS3PublisherServiceError added in v1.5.0 

func NewS3PublisherServiceError(err error) bacerrors.Error

func NewS3ResultSignerServiceError added in v1.5.0 

func NewS3ResultSignerServiceError(err error) bacerrors.Error

Types

type ClientProvider 

type ClientProvider struct {
	// contains filtered or unexported fields
}

func NewClientProvider 

func NewClientProvider(params ClientProviderParams) *ClientProvider

func (*ClientProvider) GetClient 

func (s *ClientProvider) GetClient(endpoint, region string) *ClientWrapper

func (*ClientProvider) GetConfig 

func (s *ClientProvider) GetConfig() aws.Config

GetConfig returns the AWS config used by the client provider.

func (*ClientProvider) IsInstalled 

func (s *ClientProvider) IsInstalled() bool

IsInstalled returns true if the S3 client is installed.

type ClientProviderParams 

type ClientProviderParams struct {
	AWSConfig aws.Config
}

type ClientWrapper 

type ClientWrapper struct {
	S3 *s3.Client

	Downloader *manager.Downloader
	Uploader   *manager.Uploader
	Endpoint   string
	Region     string
	// contains filtered or unexported fields
}

func (*ClientWrapper) IsAWSEndpoint added in v1.2.0 

func (c *ClientWrapper) IsAWSEndpoint() bool

IsAWSEndpoint checks if the given S3 endpoint URL is an AWS endpoint by its suffix.

func (*ClientWrapper) PresignClient added in v1.1.4 

func (c *ClientWrapper) PresignClient() *s3.PresignClient

type ObjectSummary added in v1.6.1 

type ObjectSummary struct {
	Key       *string
	ETag      *string
	VersionID *string
	Size      int64
	IsDir     bool
}

func PartitionObjects added in v1.6.1 

func PartitionObjects(
	objects []ObjectSummary,
	totalPartitions int,
	partitionIndex int,
	source SourceSpec,
) ([]ObjectSummary, error)

PartitionObjects applies the configured partitioning strategy to a slice of objects

type PartitionConfig added in v1.6.1 

type PartitionConfig struct {
	Type PartitionKeyType

	// For regex partitioning
	Pattern string

	// For substring partitioning
	StartIndex int
	EndIndex   int

	// For date partitioning
	DateFormat string
}

PartitionConfig defines how to generate partition keys from object paths

func (*PartitionConfig) Validate added in v1.6.1 

func (c *PartitionConfig) Validate() error

type PartitionKeyType added in v1.6.1 

type PartitionKeyType string

PartitionKeyType represents the type of partitioning to apply 

const (
	PartitionKeyTypeNone      PartitionKeyType = "none"
	PartitionKeyTypeObject    PartitionKeyType = "object"
	PartitionKeyTypeRegex     PartitionKeyType = "regex"
	PartitionKeyTypeSubstring PartitionKeyType = "substring"
	PartitionKeyTypeDate      PartitionKeyType = "date"
)

type PreSignedResultSpec added in v1.1.4 

type PreSignedResultSpec struct {
	SourceSpec
	PreSignedURL string
}

func DecodePreSignedResultSpec added in v1.1.4 

func DecodePreSignedResultSpec(spec *models.SpecConfig) (PreSignedResultSpec, error)

func (PreSignedResultSpec) ToMap added in v1.1.4 

func (c PreSignedResultSpec) ToMap() map[string]interface{}

func (PreSignedResultSpec) Validate added in v1.1.4 

func (c PreSignedResultSpec) Validate() error

type PublisherOption added in v1.4.0 

type PublisherOption func(p *PublisherSpec)

func WithPublisherEndpoint added in v1.4.0 

func WithPublisherEndpoint(e string) PublisherOption

func WithPublisherRegion added in v1.4.0 

func WithPublisherRegion(r string) PublisherOption

type PublisherSpec added in v1.0.4 

type PublisherSpec struct {
	Bucket   string `json:"Bucket"`
	Key      string `json:"Key"`
	Endpoint string `json:"Endpoint"`
	Region   string `json:"Region"`
}

func DecodePublisherSpec added in v1.0.4 

func DecodePublisherSpec(spec *models.SpecConfig) (PublisherSpec, error)

func (PublisherSpec) ToMap added in v1.0.4 

func (c PublisherSpec) ToMap() map[string]interface{}

func (PublisherSpec) Validate added in v1.0.4 

func (c PublisherSpec) Validate() error

type ResultSigner added in v1.1.4 

type ResultSigner struct {
	// contains filtered or unexported fields
}

func NewResultSigner added in v1.1.4 

func NewResultSigner(params ResultSignerParams) *ResultSigner

func (*ResultSigner) Transform added in v1.1.4 

func (signer *ResultSigner) Transform(ctx context.Context, spec *models.SpecConfig) error

Transform signs S3 SourceSpec with a pre-signed URL.

type ResultSignerParams added in v1.1.4 

type ResultSignerParams struct {
	ClientProvider *ClientProvider
	Expiration     time.Duration
}

type SourceSpec added in v1.0.4 

type SourceSpec struct {
	Bucket         string
	Key            string
	Filter         string
	Region         string
	Endpoint       string
	VersionID      string
	ChecksumSHA256 string
	Partition      PartitionConfig
}

func DecodeSourceSpec added in v1.0.4 

func DecodeSourceSpec(spec *models.SpecConfig) (SourceSpec, error)

func (SourceSpec) ToMap added in v1.0.4 

func (c SourceSpec) ToMap() map[string]interface{}

func (SourceSpec) Validate added in v1.0.4 

func (c SourceSpec) Validate() error

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.