s3

Documentation

Index

• type Config
  • func ParseConfig(props map[string]any) (*Config, error)
• type Connector
  • func (c Connector) ConsumeAsIterator(ctx context.Context, env *connectors.Env, source *connectors.Source, ...) (connectors.FileIterator, error)
  • func (c Connector) GetBucketMetadata(ctx context.Context, req *runtimev1.S3GetBucketMetadataRequest, ...) (string, error)
  • func (c Connector) GetCredentialsInfo(ctx context.Context, env *connectors.Env) (provider string, exist bool, err error)
  • func (c Connector) HasAnonymousAccess(ctx context.Context, env *connectors.Env, source *connectors.Source) (bool, error)
  • func (c Connector) ListBuckets(ctx context.Context, env *connectors.Env) ([]string, error)
  • func (c Connector) ListObjects(ctx context.Context, req *runtimev1.S3ListObjectsRequest, env *connectors.Env) ([]*runtimev1.S3Object, string, error)
  • func (c Connector) Spec() connectors.Spec

Types

type Config

type Config struct {
	Path                  string `mapstructure:"path"`
	AWSRegion             string `mapstructure:"region"`
	GlobMaxTotalSize      int64  `mapstructure:"glob.max_total_size"`
	GlobMaxObjectsMatched int    `mapstructure:"glob.max_objects_matched"`
	GlobMaxObjectsListed  int64  `mapstructure:"glob.max_objects_listed"`
	GlobPageSize          int    `mapstructure:"glob.page_size"`
	S3Endpoint            string `mapstructure:"endpoint"`
	// contains filtered or unexported fields
}

func ParseConfig

func ParseConfig(props map[string]any) (*Config, error)

type Connector

type Connector struct{}

func (Connector) ConsumeAsIterator

func (c Connector) ConsumeAsIterator(ctx context.Context, env *connectors.Env, source *connectors.Source, logger *zap.Logger) (connectors.FileIterator, error)

ConsumeAsIterator returns a file iterator over objects stored in gcs.

The credentials are read from following env variables

• AWS_ACCESS_KEY_ID
• AWS_SECRET_ACCESS_KEY
• AWS_SESSION_TOKEN

Additionally in case env.AllowHostCredentials is true it looks for credentials stored on host machine as well

func (Connector) GetBucketMetadata

func (c Connector) GetBucketMetadata(ctx context.Context, req *runtimev1.S3GetBucketMetadataRequest, env *connectors.Env) (string, error)

func (Connector) GetCredentialsInfo

func (c Connector) GetCredentialsInfo(ctx context.Context, env *connectors.Env) (provider string, exist bool, err error)

func (Connector) HasAnonymousAccess

func (c Connector) HasAnonymousAccess(ctx context.Context, env *connectors.Env, source *connectors.Source) (bool, error)

func (Connector) ListBuckets

func (c Connector) ListBuckets(ctx context.Context, env *connectors.Env) ([]string, error)

func (Connector) ListObjects

func (c Connector) ListObjects(ctx context.Context, req *runtimev1.S3ListObjectsRequest, env *connectors.Env) ([]*runtimev1.S3Object, string, error)

func (Connector) Spec

func (c Connector) Spec() connectors.Spec