metastore

package
v1.30.1 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Aug 7, 2024 License: Apache-2.0 Imports: 13 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

View Source 
var ErrInvalidLocation = errors.New("got empty schema or host while parsing location url, location should be schema://host/path")

Functions

func Copy added in v0.40.3 

func Copy(ctx context.Context, fromTable *Table, partitions []*Partition, toDB, toTable, serde string, setSymlink bool, toClient WriteClient, transformLocation func(location string) (string, error), fixSparkPlaceHolder bool) error

func CopyDB added in v0.54.0 

func CopyDB(ctx context.Context, fromClient, toClient Client, fromDB, toDB, toBranch string, dbfsLocation string) error

func CopyOrMerge added in v0.40.3 

func CopyOrMerge(ctx context.Context, fromClient, toClient Client, fromDB, fromTable, toDB, toTable, toBranch, serde string, partition []string, fixSparkPlaceHolder bool, dbfsLocation string) error

func CopyOrMergeAll added in v0.40.3 

func CopyOrMergeAll(ctx context.Context, fromClient, toClient Client, schemaFilter, tableFilter, toBranch string, continueOnError, fixSparkPlaceHolder bool, dbfsLocation string) error

func CopyOrMergeFromValues added in v0.40.3 

func CopyOrMergeFromValues(ctx context.Context, fromClient Client, fTable *Table, toClient Client, fromDB, fromTable, toDB, toTable, serde string, transformLocation func(location string) (string, error), fixSparkPlaceHolder bool) error
func CopyOrMergeToSymlink(ctx context.Context, fromClient, toClient Client, fromDB, fromTable, toDB, toTable, locationPrefix string, fixSparkPlaceHolder bool) error

func CopyPartition added in v0.40.3 

func CopyPartition(ctx context.Context, fromClient ReadClient, toClient Client, fromDB, fromTable, toDB, toTable, serde string, setSymlink bool, partition []string, transformLocation func(location string) (string, error), fixSparkPlaceHolder bool) error

func Diff 

func Diff(iterA, iterB Collection) (catalog.Differences, error)

func DiffIterable 

func DiffIterable(iterA, iterB Collection, callbackFn DiffCallbackFn) error

func ExtractRepoAndBranch added in v0.61.0 

func ExtractRepoAndBranch(metastoreLocationURI string) (string, string, error)

func GetSymlinkLocation 

func GetSymlinkLocation(location, locationPrefix string) (string, error)

func HandleDBFSLocation added in v0.45.1 

func HandleDBFSLocation(ctx context.Context, location string, dbfsLocation string) string

HandleDBFSLocation translates Data Bricks File system path to the S3 path using the dbfsLocation

func ImportAll added in v0.45.1 

func ImportAll(ctx context.Context, fromClient, toClient Client, schemaFilter, tableFilter, repo, toBranch string, continueOnError, fixSparkPlaceHolder bool, dbfsLocation string) error

func Merge added in v0.40.3 

func Merge(ctx context.Context, table *Table, partitionIter Collection, toDB, toTable, serde string, setSymlink bool, toClient Client, transformLocation func(location string) (string, error), fixSparkPlaceHolder bool) error

func ReplaceBranchName 

func ReplaceBranchName(location, branch string) (string, error)

func ReplaceExternalToLakeFSImported added in v0.45.1 

func ReplaceExternalToLakeFSImported(location, repo, branch string) (string, error)

Types

type Client added in v0.40.3 

type Client interface {
	ReadClient
	WriteClient
}

type Collection 

type Collection interface {
	sort.Interface
	Value(i int) interface{}
	Name(i int) string
	CompareWith(i int, v interface{}, j int) CompareResult
}

type ColumnCollection added in v0.40.3 

type ColumnCollection struct {
	// contains filtered or unexported fields
}

func NewColumnCollection added in v0.40.3 

func NewColumnCollection(columns []*FieldSchema) *ColumnCollection

func (*ColumnCollection) CompareWith added in v0.40.3 

func (c *ColumnCollection) CompareWith(i int, v interface{}, j int) CompareResult

func (*ColumnCollection) Len added in v0.40.3 

func (c *ColumnCollection) Len() int

func (*ColumnCollection) Less added in v0.40.3 

func (c *ColumnCollection) Less(i, j int) bool

func (*ColumnCollection) Name added in v0.40.3 

func (c *ColumnCollection) Name(i int) string

func (*ColumnCollection) Swap added in v0.40.3 

func (c *ColumnCollection) Swap(i, j int)

func (*ColumnCollection) Value added in v0.40.3 

func (c *ColumnCollection) Value(i int) interface{}

type CompareResult 

type CompareResult int
const (
	ItemLess CompareResult = iota
	ItemSameKey
	ItemSame
	ItemGreater
)

type Database added in v0.40.3 

type Database struct {
	Name              string
	Description       string
	LocationURI       string
	Parameters        map[string]string
	HivePrivileges    interface{}
	OwnerName         *string
	HiveOwnerType     interface{}
	AWSTargetDatabase *types.DatabaseIdentifier
}

type DiffCallbackFn 

type DiffCallbackFn func(difference catalog.DifferenceType, iter interface{}, name string) error

type FieldSchema added in v0.40.3 

type FieldSchema struct {
	Name    string
	Type    string
	Comment string
}

type MetaDiff 

type MetaDiff struct {
	PartitionDiff catalog.Differences
	ColumnsDiff   catalog.Differences
}

func GetDiff added in v0.40.3 

func GetDiff(ctx context.Context, fromClient, toClient ReadClient, fromDB, fromTable, toDB, toTable string) (*MetaDiff, error)

type Order added in v0.40.3 

type Order struct {
	Col   string
	Order int
}

type Partition added in v0.40.3 

type Partition struct {
	Values              []string
	DBName              string
	TableName           string
	CreateTime          int
	LastAccessTime      int
	Sd                  *StorageDescriptor
	Parameters          map[string]string
	AWSLastAnalyzedTime *time.Time
	Privileges          interface{}
}

func (Partition) Name added in v0.40.3 

func (m Partition) Name() string

func (*Partition) Update added in v0.40.3 

func (m *Partition) Update(ctx context.Context, db, table, serde string, setSymlink bool, transformLocation func(location string) (string, error), isSparkSQLTable, fixSparkPlaceHolder bool) error

type PartitionCollection added in v0.40.3 

type PartitionCollection struct {
	// contains filtered or unexported fields
}

func NewPartitionCollection added in v0.40.3 

func NewPartitionCollection(partitions []*Partition) *PartitionCollection

func (*PartitionCollection) CompareWith added in v0.40.3 

func (p *PartitionCollection) CompareWith(i int, v interface{}, j int) CompareResult

func (*PartitionCollection) Len added in v0.40.3 

func (p *PartitionCollection) Len() int

func (*PartitionCollection) Less added in v0.40.3 

func (p *PartitionCollection) Less(i, j int) bool

func (*PartitionCollection) Name added in v0.40.3 

func (p *PartitionCollection) Name(i int) string

func (*PartitionCollection) Swap added in v0.40.3 

func (p *PartitionCollection) Swap(i, j int)

func (*PartitionCollection) Value added in v0.40.3 

func (p *PartitionCollection) Value(i int) interface{}

type ReadClient added in v0.40.3 

type ReadClient interface {
	GetTable(ctx context.Context, dbName string, tableName string) (r *Table, err error)
	HasTable(ctx context.Context, dbName string, tableName string) (hasTable bool, err error)
	GetPartitions(ctx context.Context, dbName string, tableName string) (r []*Partition, err error)
	GetPartition(ctx context.Context, dbName string, tableName string, values []string) (r *Partition, err error)
	GetDatabase(ctx context.Context, name string) (r *Database, err error)
	GetDatabases(ctx context.Context, pattern string) (databases []*Database, err error)
	GetTables(ctx context.Context, dbName string, pattern string) (tables []*Table, err error)
}

type SerDeInfo added in v0.40.3 

type SerDeInfo struct {
	Name             string
	SerializationLib string
	Parameters       map[string]string
}

type SkewedInfo added in v0.40.3 

type SkewedInfo struct {
	SkewedColNames             []string
	SkewedColValues            [][]string
	AWSSkewedColValues         []string //
	SkewedColValueLocationMaps map[string]string
}

type StorageDescriptor added in v0.40.3 

type StorageDescriptor struct {
	Cols                   []*FieldSchema
	Location               string
	InputFormat            string
	OutputFormat           string
	Compressed             bool
	NumBuckets             int
	SerdeInfo              *SerDeInfo
	BucketCols             []string
	SortCols               []*Order
	Parameters             map[string]string
	SkewedInfo             *SkewedInfo
	StoredAsSubDirectories *bool
	AWSSchemaReference     interface{}
}

func (*StorageDescriptor) Update added in v0.41.1 

func (m *StorageDescriptor) Update(db, table, serde string, setSymlink bool, transformLocation func(location string) (string, error), isSparkSQLTable, fixSparkPlaceHolder bool) error

type Table added in v0.40.3 

type Table struct {
	TableName                        string
	DBName                           string
	Owner                            string
	CreateTime                       int64
	LastAccessTime                   int64
	Retention                        int
	Sd                               *StorageDescriptor
	PartitionKeys                    []*FieldSchema
	Parameters                       map[string]string
	ViewOriginalText                 string
	ViewExpandedText                 string
	TableType                        string
	Temporary                        bool
	RewriteEnabled                   *bool
	AWSCreatedBy                     *string
	AWSDescription                   *string
	AWSIsRegisteredWithLakeFormation *bool
	AWSLastAnalyzedTime              *time.Time
	AWSTargetTable                   interface{}
	AWSUpdateTime                    *time.Time
	Privileges                       interface{}
}

func (*Table) Update added in v0.40.3 

func (m *Table) Update(ctx context.Context, db, table, serde string, setSymlink bool, transformLocation func(location string) (string, error), isSparkSQLTable, fixSparkPlaceHolder bool) error

type WriteClient added in v0.40.3 

type WriteClient interface {
	CreateTable(ctx context.Context, tbl *Table) error
	AlterTable(ctx context.Context, dbName string, tableName string, newTable *Table) error
	AddPartitions(ctx context.Context, tableName string, dbName string, newParts []*Partition) error
	AlterPartitions(ctx context.Context, dbName string, tableName string, newPartitions []*Partition) error
	AlterPartition(ctx context.Context, dbName string, tableName string, partition *Partition) error
	AddPartition(ctx context.Context, tableName string, dbName string, newPartition *Partition) error
	DropPartition(ctx context.Context, dbName string, tableName string, values []string) error
	CreateDatabase(ctx context.Context, database *Database) error
	NormalizeDBName(name string) string // NormalizeDBName changes the db name to be a valid name for the client
	GetDBLocation(dbName string) string // getDBLocation returns the expected locationURI of the database
}

Source Files

View all Source files

Directories

Path Synopsis
gen-go/hive_metastore
gen-go/hive_metastore/thrift_hive_metastore-remote

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.