gcs

Documentation

Index

• type Config
  • func ParseConfig(props map[string]any) (*Config, error)
• type Connector
  • func (c Connector) ConsumeAsIterator(ctx context.Context, env *connectors.Env, source *connectors.Source, ...) (connectors.FileIterator, error)
  • func (c Connector) GetCredentialsInfo(ctx context.Context, env *connectors.Env) (string, bool, error)
  • func (c Connector) HasAnonymousAccess(ctx context.Context, env *connectors.Env, source *connectors.Source) (bool, error)
  • func (c Connector) ListBuckets(ctx context.Context, req *runtimev1.GCSListBucketsRequest, env *connectors.Env) ([]string, string, error)
  • func (c Connector) ListObjects(ctx context.Context, req *runtimev1.GCSListObjectsRequest, env *connectors.Env) ([]*runtimev1.GCSObject, string, error)
  • func (c Connector) Spec() connectors.Spec

Types

type Config

type Config struct {
	Path                  string `key:"path"`
	GlobMaxTotalSize      int64  `mapstructure:"glob.max_total_size"`
	GlobMaxObjectsMatched int    `mapstructure:"glob.max_objects_matched"`
	GlobMaxObjectsListed  int64  `mapstructure:"glob.max_objects_listed"`
	GlobPageSize          int    `mapstructure:"glob.page_size"`
	// contains filtered or unexported fields
}

func ParseConfig

func ParseConfig(props map[string]any) (*Config, error)

type Connector

type Connector struct{}

func (Connector) ConsumeAsIterator

func (c Connector) ConsumeAsIterator(ctx context.Context, env *connectors.Env, source *connectors.Source, l *zap.Logger) (connectors.FileIterator, error)

ConsumeAsIterator returns a file iterator over objects stored in gcs. The credential json is read from a env variable google_application_credentials. Additionally in case `env.AllowHostCredentials` is true it looks for "Application Default Credentials" as well

func (Connector) GetCredentialsInfo

func (c Connector) GetCredentialsInfo(ctx context.Context, env *connectors.Env) (string, bool, error)

func (Connector) HasAnonymousAccess

func (c Connector) HasAnonymousAccess(ctx context.Context, env *connectors.Env, source *connectors.Source) (bool, error)

func (Connector) ListBuckets

func (c Connector) ListBuckets(ctx context.Context, req *runtimev1.GCSListBucketsRequest, env *connectors.Env) ([]string, string, error)

func (Connector) ListObjects

func (c Connector) ListObjects(ctx context.Context, req *runtimev1.GCSListObjectsRequest, env *connectors.Env) ([]*runtimev1.GCSObject, string, error)

func (Connector) Spec

func (c Connector) Spec() connectors.Spec