profiles

package
v1.0.1-0...-bc21bb4 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jan 13, 2025 License: Apache-2.0 Imports: 13 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	SourceProfileTypeUnset = iota
	SourceProfileTypeFile
	SourceProfileTypeConnection
	SourceProfileTypeConfig
	SourceProfileTypeCsv
	SourceProfileTypeCloudSQL
)
View Source 
const (
	SourceProfileConnectionTypeUnset = iota
	SourceProfileConnectionTypeMySQL
	SourceProfileConnectionTypePostgreSQL
	SourceProfileConnectionTypeDynamoDB
	SourceProfileConnectionTypeSqlServer
	SourceProfileConnectionTypeOracle
)
View Source 
const (
	SourceProfileConnectionTypeCloudSQLUnset = iota
	SourceProfileConnectionTypeCloudSQLMySQL
	SourceProfileConnectionTypeCloudSQLPostgreSQL
)
View Source 
const (
	TargetProfileTypeUnset = iota
	TargetProfileTypeConnection
)
View Source 
const (
	TargetProfileConnectionTypeUnset = iota
	TargetProfileConnectionTypeSpanner
)

Variables

This section is empty.

Functions

func GenerateMYSQLConnectionStr 

func GenerateMYSQLConnectionStr() (string, error)

func GeneratePGSQLConnectionStr 

func GeneratePGSQLConnectionStr() (string, error)

func GetSQLConnectionStr 

func GetSQLConnectionStr(sourceProfile SourceProfile) string

func GetSchemaSampleSize 

func GetSchemaSampleSize(sourceProfile SourceProfile) int64

func ParseList 

func ParseList(s string) ([]string, error)

func ParseMap 

func ParseMap(s string) (map[string]string, error)

Parses input string `s` as a map of key-value pairs. It's expected that the input string is of the form "key1=value1,key2=value2,..." etc. Return error otherwise.

Types

type DataShard 

type DataShard struct {
	DataShardId          string                      `json:"dataShardId"`
	SrcConnectionProfile DatastreamConnProfileSource `json:"srcConnectionProfile"`
	DstConnectionProfile DatastreamConnProfileTarget `json:"dstConnectionProfile"`
	DatastreamConfig     DatastreamConfig            `json:"datastreamConfig"`
	GcsConfig            GcsConfig                   `json:"gcsConfig"`
	DataflowConfig       DataflowConfig              `json:"dataflowConfig"`
	TmpDir               string                      `json:"tmpDir"`
	StreamLocation       string                      `json:"streamLocation"`
	LogicalShards        []LogicalShard              `json:"databases"`
}

type DataflowConfig 

type DataflowConfig struct {
	ProjectId            string `json:"projectId"`
	Location             string `json:"location"`
	Network              string `json:"network"`
	Subnetwork           string `json:"subnetwork"`
	VpcHostProjectId     string `json:"hostProjectId"`
	MaxWorkers           string `json:"maxWorkers"`
	NumWorkers           string `json:"numWorkers"`
	ServiceAccountEmail  string `json:"serviceAccountEmail"`
	JobName              string `json:"jobName"`
	MachineType          string `json:"machineType"`
	AdditionalUserLabels string `json:"additionalUserLabels"`
	KmsKeyName           string `json:"kmsKeyName"`
	GcsTemplatePath      string `json:"gcsTemplatePath"`
	CustomJarPath        string `json:"customJarPath"`
	CustomClassName      string `json:"customClassName"`
	CustomParameter      string `json:"customParameter"`
}

type DatastreamConfig 

type DatastreamConfig struct {
	MaxConcurrentBackfillTasks string `json:"maxConcurrentBackfillTasks"`
	MaxConcurrentCdcTasks      string `json:"maxConcurrentCdcTasks"`
}

type DatastreamConnProfileSource 

type DatastreamConnProfileSource struct {
	Name     string `json:"name"`
	Host     string `json:"host"`
	User     string `json:"user"`
	Port     string `json:"port"`
	Password string `json:"password"`
	Location string `json:"location"`
}

type DatastreamConnProfileTarget 

type DatastreamConnProfileTarget struct {
	Name     string `json:"name"`
	Location string `json:"location"`
}

type DirectConnectionConfig 

type DirectConnectionConfig struct {
	DataShardId string `json:"dataShardId"`
	Host        string `json:"host"`
	User        string `json:"user"`
	Password    string `json:"password"`
	Port        string `json:"port"`
	DbName      string `json:"dbName"`
}

type GcsConfig 

type GcsConfig struct {
	TtlInDays    int64 `json:"ttlInDays,string"`
	TtlInDaysSet bool  `json:"ttlInDaysSet"`
}

type LogicalShard 

type LogicalShard struct {
	DbName         string `json:"dbName"`
	LogicalShardId string `json:"databaseId"`
	RefDataShardId string `json:"refDataShardId"`
}

type NewSourceProfileImpl 

type NewSourceProfileImpl struct{}

func (*NewSourceProfileImpl) NewSourceProfileConfig 

func (nsp *NewSourceProfileImpl) NewSourceProfileConfig(source string, path string) (SourceProfileConfig, error)

func (*NewSourceProfileImpl) NewSourceProfileConnection 

func (nsp *NewSourceProfileImpl) NewSourceProfileConnection(source string, params map[string]string, s SourceProfileDialectInterface) (SourceProfileConnection, error)

func (*NewSourceProfileImpl) NewSourceProfileConnectionCloudSQL 

func (nsp *NewSourceProfileImpl) NewSourceProfileConnectionCloudSQL(source string, params map[string]string, s SourceProfileDialectInterface) (SourceProfileConnectionCloudSQL, error)

func (*NewSourceProfileImpl) NewSourceProfileFile 

func (nsp *NewSourceProfileImpl) NewSourceProfileFile(params map[string]string) SourceProfileFile

type NewSourceProfileInterface 

type NewSourceProfileInterface interface {
	NewSourceProfileFile(params map[string]string) SourceProfileFile
	NewSourceProfileConfig(source string, path string) (SourceProfileConfig, error)
	NewSourceProfileConnectionCloudSQL(source string, params map[string]string, s SourceProfileDialectInterface) (SourceProfileConnectionCloudSQL, error)
	NewSourceProfileConnection(source string, params map[string]string, s SourceProfileDialectInterface) (SourceProfileConnection, error)
}

Interface to create new source profiles for different input types

type ShardConfigurationBulk 

type ShardConfigurationBulk struct {
	SchemaSource DirectConnectionConfig   `json:"schemaSource"`
	DataShards   []DirectConnectionConfig `json:"dataShards"`
}

type ShardConfigurationDMS 

type ShardConfigurationDMS struct {
}

TODO: Define the sharding structure for DMS migrations here.

type ShardConfigurationDataflow 

type ShardConfigurationDataflow struct {
	SchemaSource     DirectConnectionConfig `json:"schemaSource"`
	DataShards       []*DataShard           `json:"dataShards"`
	DatastreamConfig DatastreamConfig       `json:"datastreamConfig"`
	GcsConfig        GcsConfig              `json:"gcsConfig"`
	DataflowConfig   DataflowConfig         `json:"dataflowConfig"`
}

type SourceProfile 

type SourceProfile struct {
	Driver       string
	Ty           SourceProfileType
	File         SourceProfileFile
	Conn         SourceProfileConnection
	ConnCloudSQL SourceProfileConnectionCloudSQL
	Config       SourceProfileConfig
	Csv          SourceProfileCsv
}

func NewSourceProfile 

func NewSourceProfile(s string, source string, n NewSourceProfileInterface) (SourceProfile, error)

Flag source-profile is passed as a list of key value pairs on the command line. Following 3 formats are supported as a valid source-profile.

Format 1. Specify file path and file format. File path can be a local file path or a gcs file path. Support for more file path types can be added in future. File format can be "dump" e.g., when specifying a mysqldump or pgdump etc. Support for more formats e.g., "csv", "avro" etc can be added in future.

Example: -source-profile="file=/tmp/abc, format=dump" Example: -source-profile="file=gcs://bucket_name/cart.txt, format=dump"

Format 2. Specify source connection parameters. If none specified, then read from envrironment variables.

Format 3. Specify a config file that specifies source connection profile.

func (SourceProfile) ToLegacyDriver 

func (src SourceProfile) ToLegacyDriver(source string) (string, error)

ToLegacyDriver converts source-profile to equivalent legacy global flags e.g., -driver, -dump-file etc since the rest of the codebase still uses the same. TODO: Deprecate this function and pass around SourceProfile across the codebase wherever information about source connection is required.

func (SourceProfile) UseTargetSchema 

func (src SourceProfile) UseTargetSchema() bool

UseTargetSchema returns true if the driver expects an existing schema to use in the target database.

type SourceProfileConfig 

type SourceProfileConfig struct {
	ConfigType                 string                     `json:"configType"`
	ShardConfigurationBulk     ShardConfigurationBulk     `json:"shardConfigurationBulk"`
	ShardConfigurationDataflow ShardConfigurationDataflow `json:"shardConfigurationDataflow"`
	ShardConfigurationDMS      ShardConfigurationDMS      `json:"shardConfigurationDMS"`
}

type SourceProfileConnection 

type SourceProfileConnection struct {
	Ty        SourceProfileConnectionType
	Streaming bool
	Mysql     SourceProfileConnectionMySQL
	Pg        SourceProfileConnectionPostgreSQL
	Dydb      SourceProfileConnectionDynamoDB
	SqlServer SourceProfileConnectionSqlServer
	Oracle    SourceProfileConnectionOracle
}

type SourceProfileConnectionCloudSQL 

type SourceProfileConnectionCloudSQL struct {
	Ty    SourceProfileConnectionTypeCloudSQL
	Mysql SourceProfileConnectionCloudSQLMySQL
	Pg    SourceProfileConnectionCloudSQLPostgreSQL
}

type SourceProfileConnectionCloudSQLMySQL 

type SourceProfileConnectionCloudSQLMySQL struct {
	User         string
	Db           string
	InstanceName string
	Project      string
	Region       string
}

type SourceProfileConnectionCloudSQLPostgreSQL 

type SourceProfileConnectionCloudSQLPostgreSQL struct {
	User         string
	Db           string
	InstanceName string
	Project      string
	Region       string
}

type SourceProfileConnectionDynamoDB 

type SourceProfileConnectionDynamoDB struct {
	// These connection params are not used currently because the SDK reads directly from the env variables.
	// These are still kept around as reference when we refactor passing
	// SourceProfile instead of sqlConnectionStr around.
	AwsAccessKeyID     string // Same as AWS_ACCESS_KEY_ID environment variable
	AwsSecretAccessKey string // Same as AWS_SECRET_ACCESS_KEY environment variable
	AwsRegion          string // Same as AWS_REGION environment variable
	DydbEndpoint       string // Same as DYNAMODB_ENDPOINT_OVERRIDE environment variable
	SchemaSampleSize   int64  // Number of rows to use for inferring schema (default 100,000)
	// contains filtered or unexported fields
}

type SourceProfileConnectionMySQL 

type SourceProfileConnectionMySQL struct {
	Host            string // Same as MYSQLHOST environment variable
	Port            string // Same as MYSQLPORT environment variable
	User            string // Same as MYSQLUSER environment variable
	Db              string // Same as MYSQLDATABASE environment variable
	Pwd             string // Same as MYSQLPWD environment variable
	StreamingConfig string
}

type SourceProfileConnectionOracle 

type SourceProfileConnectionOracle struct {
	Host            string
	Port            string
	User            string
	Db              string
	Pwd             string
	StreamingConfig string
}

type SourceProfileConnectionPostgreSQL 

type SourceProfileConnectionPostgreSQL struct {
	Host            string // Same as PGHOST environment variable
	Port            string // Same as PGPORT environment variable
	User            string // Same as PGUSER environment variable
	Db              string // Same as PGDATABASE environment variable
	Pwd             string // Same as PGPASSWORD environment variable
	StreamingConfig string
}

type SourceProfileConnectionSqlServer 

type SourceProfileConnectionSqlServer struct {
	Host string
	Port string
	User string
	Db   string
	Pwd  string
}

type SourceProfileConnectionType 

type SourceProfileConnectionType int

type SourceProfileConnectionTypeCloudSQL 

type SourceProfileConnectionTypeCloudSQL int

type SourceProfileCsv 

type SourceProfileCsv struct {
	Manifest  string
	Delimiter string
	NullStr   string
}

func NewSourceProfileCsv 

func NewSourceProfileCsv(params map[string]string) SourceProfileCsv

type SourceProfileDialectImpl 

type SourceProfileDialectImpl struct{}

func (*SourceProfileDialectImpl) NewSourceProfileConnectionCloudSQLMySQL 

func (spd *SourceProfileDialectImpl) NewSourceProfileConnectionCloudSQLMySQL(params map[string]string, g utils.GetUtilInfoInterface) (SourceProfileConnectionCloudSQLMySQL, error)

func (*SourceProfileDialectImpl) NewSourceProfileConnectionCloudSQLPostgreSQL 

func (spd *SourceProfileDialectImpl) NewSourceProfileConnectionCloudSQLPostgreSQL(params map[string]string, g utils.GetUtilInfoInterface) (SourceProfileConnectionCloudSQLPostgreSQL, error)

func (*SourceProfileDialectImpl) NewSourceProfileConnectionDynamoDB 

func (spd *SourceProfileDialectImpl) NewSourceProfileConnectionDynamoDB(params map[string]string, g utils.GetUtilInfoInterface) (SourceProfileConnectionDynamoDB, error)

func (*SourceProfileDialectImpl) NewSourceProfileConnectionMySQL 

func (spd *SourceProfileDialectImpl) NewSourceProfileConnectionMySQL(params map[string]string, g utils.GetUtilInfoInterface) (SourceProfileConnectionMySQL, error)

func (*SourceProfileDialectImpl) NewSourceProfileConnectionOracle 

func (spd *SourceProfileDialectImpl) NewSourceProfileConnectionOracle(params map[string]string, g utils.GetUtilInfoInterface) (SourceProfileConnectionOracle, error)

func (*SourceProfileDialectImpl) NewSourceProfileConnectionPostgreSQL 

func (spd *SourceProfileDialectImpl) NewSourceProfileConnectionPostgreSQL(params map[string]string, g utils.GetUtilInfoInterface) (SourceProfileConnectionPostgreSQL, error)

func (*SourceProfileDialectImpl) NewSourceProfileConnectionSqlServer 

func (spd *SourceProfileDialectImpl) NewSourceProfileConnectionSqlServer(params map[string]string, g utils.GetUtilInfoInterface) (SourceProfileConnectionSqlServer, error)

type SourceProfileDialectInterface 

type SourceProfileDialectInterface interface {
	NewSourceProfileConnectionCloudSQLMySQL(params map[string]string, g utils.GetUtilInfoInterface) (SourceProfileConnectionCloudSQLMySQL, error)
	NewSourceProfileConnectionMySQL(params map[string]string, g utils.GetUtilInfoInterface) (SourceProfileConnectionMySQL, error)
	NewSourceProfileConnectionCloudSQLPostgreSQL(params map[string]string, g utils.GetUtilInfoInterface) (SourceProfileConnectionCloudSQLPostgreSQL, error)
	NewSourceProfileConnectionPostgreSQL(params map[string]string, g utils.GetUtilInfoInterface) (SourceProfileConnectionPostgreSQL, error)
	NewSourceProfileConnectionSqlServer(params map[string]string, g utils.GetUtilInfoInterface) (SourceProfileConnectionSqlServer, error)
	NewSourceProfileConnectionDynamoDB(params map[string]string, g utils.GetUtilInfoInterface) (SourceProfileConnectionDynamoDB, error)
	NewSourceProfileConnectionOracle(params map[string]string, g utils.GetUtilInfoInterface) (SourceProfileConnectionOracle, error)
}

Interface to create source profiles for different database dialects

type SourceProfileFile 

type SourceProfileFile struct {
	Path   string
	Format string
}

type SourceProfileType 

type SourceProfileType int

type TargetProfile 

type TargetProfile struct {
	Ty   TargetProfileType
	Conn TargetProfileConnection
}

func NewTargetProfile 

func NewTargetProfile(s string) (TargetProfile, error)

Target profile is passed as a list of key value pairs on the command line. Today we support only direct connection as a valid target profile type, but in future we can support writing to CSV or AVRO as valid targets.

Among direct connection targets, today we only support Spanner database. TargetProfileConnectionType can be extended to add more databases. Users can specify the database dialect, instance, database name etc when connecting to Spanner.

Database dialect can take 2 values: GoogleSQL or PostgreSQL and the same correspond to regular Cloud Spanner database and PG Cloud Spanner database respectively.

If dbName is not specified, then Spanner migration tool will autogenerate the same and create a database with the same name.

Example: -target-profile="instance=my-instance1,dbName=my-new-db1" Example: -target-profile="instance=my-instance1,dbName=my-new-db1,dialect=PostgreSQL"

func (TargetProfile) FetchTargetDialect 

func (trg TargetProfile) FetchTargetDialect(ctx context.Context) (string, error)

This expects that GetResourceIds has already been called once and the project, instance and dbName fields in target profile are populated.

func (*TargetProfile) GetResourceIds 

func (targetProfile *TargetProfile) GetResourceIds(ctx context.Context, now time.Time, driverName string, out *os.File, g utils.GetUtilInfoInterface) (string, string, string, error)

type TargetProfileConnection 

type TargetProfileConnection struct {
	Ty TargetProfileConnectionType
	Sp TargetProfileConnectionSpanner
}

type TargetProfileConnectionSpanner 

type TargetProfileConnectionSpanner struct {
	Endpoint string // Same as SPANNER_API_ENDPOINT environment variable
	Project  string // Same as GCLOUD_PROJECT environment variable
	Instance string
	Dbname   string
	Dialect  string
}

type TargetProfileConnectionType 

type TargetProfileConnectionType int

type TargetProfileType 

type TargetProfileType int

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.