data_source

Documentation

Index

• func ClearMetadata()
• func GetDataObjectType(name string, cfg *config.ConfigMap) *ds.DataObjectType
• func GetS3MetaData(cfg *config.ConfigMap) *ds.MetaData
• type AwsGlueRepository
  • func NewAwsGlueRepository(configMap *config.ConfigMap) *AwsGlueRepository
  • func (repo *AwsGlueRepository) GetGlueClient(ctx context.Context, region *string) (*glue.Client, error)
  • func (repo *AwsGlueRepository) ListDatabases(ctx context.Context, accountId string, region string) ([]string, error)
  • func (repo *AwsGlueRepository) ListTablesForDatabase(ctx context.Context, accountId string, database string, region string) (map[string]string, error)
• type AwsS3Repository
  • func NewAwsS3Repository(configMap *config.ConfigMap) *AwsS3Repository
  • func (repo *AwsS3Repository) GetFile(ctx context.Context, bucket, key string, region *string) (io.ReadCloser, error)
  • func (repo *AwsS3Repository) GetS3Client(ctx context.Context, region *string) (*s3.Client, error)
  • func (repo *AwsS3Repository) ListBuckets(ctx context.Context) ([]model.AwsS3Entity, error)
  • func (repo *AwsS3Repository) ListFiles(ctx context.Context, bucket string, prefix *string) ([]model.AwsS3Entity, string, error)
• type DataSourceSyncer
  • func NewDataSourceSyncer() *DataSourceSyncer
  • func (s *DataSourceSyncer) FetchGlueDataObjects(ctx context.Context, dataSourceHandler wrappers.DataSourceObjectHandler, ...) error
  • func (s *DataSourceSyncer) FetchS3DataObjects(ctx context.Context, dataSourceHandler wrappers.DataSourceObjectHandler) error
  • func (s *DataSourceSyncer) GetAvailableObjectTypes(ctx context.Context, cfg *config.ConfigMap) (*trie.Trie[string], error)
  • func (s *DataSourceSyncer) GetDataSourceMetaData(ctx context.Context, configParams *config.ConfigMap) (*ds.MetaData, error)
  • func (s *DataSourceSyncer) SyncDataSource(ctx context.Context, dataSourceHandler wrappers.DataSourceObjectHandler, ...) error
• type MetadataProvider
• type S3FileMetadataProvider
  • func (p S3FileMetadataProvider) AccessProviderTypes() []*ds.AccessProviderType
  • func (p S3FileMetadataProvider) DataObjectTypes() []*ds.DataObjectType
  • func (p S3FileMetadataProvider) UsageMetaInfo() *ds.UsageMetaInput
• type S3GlueMetadataProvider
  • func (p S3GlueMetadataProvider) AccessProviderTypes() []*ds.AccessProviderType
  • func (p S3GlueMetadataProvider) DataObjectTypes() []*ds.DataObjectType
  • func (p S3GlueMetadataProvider) UsageMetaInfo() *ds.UsageMetaInput

Functions

func ClearMetadata

func ClearMetadata()

func GetDataObjectType

func GetDataObjectType(name string, cfg *config.ConfigMap) *ds.DataObjectType

func GetS3MetaData

func GetS3MetaData(cfg *config.ConfigMap) *ds.MetaData

Types

type AwsGlueRepository

type AwsGlueRepository struct {
	// contains filtered or unexported fields
}

func NewAwsGlueRepository

func NewAwsGlueRepository(configMap *config.ConfigMap) *AwsGlueRepository

func (*AwsGlueRepository) GetGlueClient

func (repo *AwsGlueRepository) GetGlueClient(ctx context.Context, region *string) (*glue.Client, error)

func (*AwsGlueRepository) ListDatabases

func (repo *AwsGlueRepository) ListDatabases(ctx context.Context, accountId string, region string) ([]string, error)

func (*AwsGlueRepository) ListTablesForDatabase

func (repo *AwsGlueRepository) ListTablesForDatabase(ctx context.Context, accountId string, database string, region string) (map[string]string, error)

type AwsS3Repository

type AwsS3Repository struct {
	// contains filtered or unexported fields
}

func NewAwsS3Repository

func NewAwsS3Repository(configMap *config.ConfigMap) *AwsS3Repository

func (*AwsS3Repository) GetFile

func (repo *AwsS3Repository) GetFile(ctx context.Context, bucket, key string, region *string) (io.ReadCloser, error)

func (*AwsS3Repository) GetS3Client

func (repo *AwsS3Repository) GetS3Client(ctx context.Context, region *string) (*s3.Client, error)

func (*AwsS3Repository) ListBuckets

func (repo *AwsS3Repository) ListBuckets(ctx context.Context) ([]model.AwsS3Entity, error)

func (*AwsS3Repository) ListFiles

func (repo *AwsS3Repository) ListFiles(ctx context.Context, bucket string, prefix *string) ([]model.AwsS3Entity, string, error)

type DataSourceSyncer

type DataSourceSyncer struct {
	// contains filtered or unexported fields
}

func NewDataSourceSyncer

func NewDataSourceSyncer() *DataSourceSyncer

func (*DataSourceSyncer) FetchGlueDataObjects

func (s *DataSourceSyncer) FetchGlueDataObjects(ctx context.Context, dataSourceHandler wrappers.DataSourceObjectHandler, region string) error

func (*DataSourceSyncer) FetchS3DataObjects

func (s *DataSourceSyncer) FetchS3DataObjects(ctx context.Context, dataSourceHandler wrappers.DataSourceObjectHandler) error

func (*DataSourceSyncer) GetAvailableObjectTypes

func (s *DataSourceSyncer) GetAvailableObjectTypes(ctx context.Context, cfg *config.ConfigMap) (*trie.Trie[string], error)

GetAvailableObjectTypes is used by the data usage component to fetch all available data objects and corresponding type

func (*DataSourceSyncer) GetDataSourceMetaData

func (s *DataSourceSyncer) GetDataSourceMetaData(ctx context.Context, configParams *config.ConfigMap) (*ds.MetaData, error)

func (*DataSourceSyncer) SyncDataSource

func (s *DataSourceSyncer) SyncDataSource(ctx context.Context, dataSourceHandler wrappers.DataSourceObjectHandler, cfg *ds.DataSourceSyncConfig) error

type MetadataProvider

type MetadataProvider interface {
	DataObjectTypes() []*ds.DataObjectType
	UsageMetaInfo() *ds.UsageMetaInput
	AccessProviderTypes() []*ds.AccessProviderType
}

type S3FileMetadataProvider

type S3FileMetadataProvider struct {
}

func (S3FileMetadataProvider) AccessProviderTypes

func (p S3FileMetadataProvider) AccessProviderTypes() []*ds.AccessProviderType

func (S3FileMetadataProvider) DataObjectTypes

func (p S3FileMetadataProvider) DataObjectTypes() []*ds.DataObjectType

func (S3FileMetadataProvider) UsageMetaInfo

func (p S3FileMetadataProvider) UsageMetaInfo() *ds.UsageMetaInput

type S3GlueMetadataProvider

type S3GlueMetadataProvider struct {
}

func (S3GlueMetadataProvider) AccessProviderTypes

func (p S3GlueMetadataProvider) AccessProviderTypes() []*ds.AccessProviderType

func (S3GlueMetadataProvider) DataObjectTypes

func (p S3GlueMetadataProvider) DataObjectTypes() []*ds.DataObjectType

func (S3GlueMetadataProvider) UsageMetaInfo

func (p S3GlueMetadataProvider) UsageMetaInfo() *ds.UsageMetaInput