conversion

package
v1.0.1-0...-b7ad348 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Dec 7, 2024 License: Apache-2.0 Imports: 57 Imported by: 0

Documentation

Overview

TODO:(searce) Organize code in go style format to make this file more readable. 

public constants first
key public type definitions next (although often it makes sense to put them next to public functions that use them)
then public functions (and relevant type definitions)
and helper functions and other non-public definitions last (generally in order of importance)

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

func GetBucketFromDatastreamProfile 

func GetBucketFromDatastreamProfile(project, location, profileName string) (string, string, error)

func GetDatastreamClient 

func GetDatastreamClient(ctx context.Context) *datastream.Client

func ReadSessionFile 

func ReadSessionFile(conv *internal.Conv, sessionJSON string) error

ReadSessionFile reads a session JSON file and unmarshal it's content into *internal.Conv.

func ValidateTables 

func ValidateTables(ctx context.Context, client *sp.Client, spDialect string) (string, error)

ValidateTables validates that all the tables in the database are empty. It returns the name of the first non-empty table if found, and an empty string otherwise.

func WriteBadData 

func WriteBadData(bw *writer.BatchWriter, conv *internal.Conv, banner, name string, out *os.File)

WriteBadData prints summary stats about bad rows and writes detailed info to file 'name'.

func WriteConvGeneratedFiles 

func WriteConvGeneratedFiles(conv *internal.Conv, dbName string, driver string, BytesRead int64, out *os.File) (string, error)

WriteConvGeneratedFiles creates a directory labeled downloads with the current timestamp where it writes the sessionfile, report summary and DDLs then returns the directory where it writes.

func WriteSchemaFile 

func WriteSchemaFile(conv *internal.Conv, now time.Time, name string, out *os.File, driver string)

WriteSchemaFile writes DDL statements in a file. It includes CREATE TABLE statements and ALTER TABLE statements to add foreign keys. The parameter name should end with a .txt.

func WriteSessionFile 

func WriteSessionFile(conv *internal.Conv, name string, out *os.File)

WriteSessionFile writes conv struct to a file in JSON format.

Types

type ConnectionProfile 

type ConnectionProfile struct {
	// Project Id of the resource
	ProjectId string
	// Datashard Id for the resource
	DatashardId string
	// Name of connection profile
	Id string
	// If true, don't create resource, only validate if creation is possible. If false, create resource.
	ValidateOnly bool
	// If true, create source connection profile, else create target connection profile and gcs bucket.
	IsSource bool
	// For source connection profile host of MySql instance
	Host string
	// For source connection profile port of MySql instance
	Port string
	// For source connection profile password of MySql instance
	Password string
	// For source connection profile user name of MySql instance
	User string
	// Region of connection profile to be created
	Region string
	// For target connection profile name of gcs bucket to be created
	BucketName string
}

type ConnectionProfileReq 

type ConnectionProfileReq struct {
	ConnectionProfile ConnectionProfile
	Error             error
	Ctx               context.Context
}

type ConvImpl 

type ConvImpl struct{}

func (*ConvImpl) DataConv 

func (ci *ConvImpl) DataConv(ctx context.Context, migrationProjectId string, sourceProfile profiles.SourceProfile, targetProfile profiles.TargetProfile, ioHelper *utils.IOStreams, client *sp.Client, conv *internal.Conv, dataOnly bool, writeLimit int64, dataFromSource DataFromSourceInterface) (*writer.BatchWriter, error)

DataConv performs the data conversion The SourceProfile param provides the connection details to use the go SQL library.

func (*ConvImpl) SchemaConv 

func (ci *ConvImpl) SchemaConv(migrationProjectId string, sourceProfile profiles.SourceProfile, targetProfile profiles.TargetProfile, ioHelper *utils.IOStreams, schemaFromSource SchemaFromSourceInterface) (*internal.Conv, error)

SchemaConv performs the schema conversion The SourceProfile param provides the connection details to use the go SQL library.

type ConvInterface 

type ConvInterface interface {
	SchemaConv(migrationProjectId string, sourceProfile profiles.SourceProfile, targetProfile profiles.TargetProfile, ioHelper *utils.IOStreams, schemaFromSource SchemaFromSourceInterface) (*internal.Conv, error)
	DataConv(ctx context.Context, migrationProjectId string, sourceProfile profiles.SourceProfile, targetProfile profiles.TargetProfile, ioHelper *utils.IOStreams, client *sp.Client, conv *internal.Conv, dataOnly bool, writeLimit int64, dataFromSource DataFromSourceInterface) (*writer.BatchWriter, error)
}

type CreateMigrationResources 

type CreateMigrationResources interface {
	// contains filtered or unexported methods
}

type DataFromDatabaseImpl 

type DataFromDatabaseImpl struct{}

type DataFromDatabaseInterface 

type DataFromDatabaseInterface interface {
	// contains filtered or unexported methods
}

type DataFromSourceImpl 

type DataFromSourceImpl struct{}

type DataFromSourceInterface 

type DataFromSourceInterface interface {
	// contains filtered or unexported methods
}

type GetInfoImpl 

type GetInfoImpl struct{}

func (*GetInfoImpl) GetInfoSchema 

func (gi *GetInfoImpl) GetInfoSchema(migrationProjectId string, sourceProfile profiles.SourceProfile, targetProfile profiles.TargetProfile) (common.InfoSchema, error)

func (*GetInfoImpl) GetInfoSchemaFromCloudSQL 

func (gi *GetInfoImpl) GetInfoSchemaFromCloudSQL(migrationProjectId string, sourceProfile profiles.SourceProfile, targetProfile profiles.TargetProfile) (common.InfoSchema, error)

type GetInfoInterface 

type GetInfoInterface interface {
	GetInfoSchemaFromCloudSQL(migrationProjectId string, sourceProfile profiles.SourceProfile, targetProfile profiles.TargetProfile) (common.InfoSchema, error)
	GetInfoSchema(migrationProjectId string, sourceProfile profiles.SourceProfile, targetProfile profiles.TargetProfile) (common.InfoSchema, error)
	// contains filtered or unexported methods
}

type MockDataFromSource 

type MockDataFromSource struct {
	mock.Mock
}

type MockGetInfo 

type MockGetInfo struct {
	mock.Mock
}

func (*MockGetInfo) GetInfoSchema 

func (mgi *MockGetInfo) GetInfoSchema(migrationShardId string, sourceProfile profiles.SourceProfile, targetProfile profiles.TargetProfile) (common.InfoSchema, error)

func (*MockGetInfo) GetInfoSchemaFromCloudSQL 

func (mgi *MockGetInfo) GetInfoSchemaFromCloudSQL(migrationShardId string, sourceProfile profiles.SourceProfile, targetProfile profiles.TargetProfile) (common.InfoSchema, error)

type MockResourceGeneration 

type MockResourceGeneration struct {
	mock.Mock
}

func (*MockResourceGeneration) GetConnectionProfilesForResources 

func (mrg *MockResourceGeneration) GetConnectionProfilesForResources(ctx context.Context, projectId string, sourceProfile profiles.SourceProfile, region string, validateOnly bool) ([]*ConnectionProfileReq, []*ConnectionProfileReq, error)

func (*MockResourceGeneration) PrepareMinimalDowntimeResources 

func (mrg *MockResourceGeneration) PrepareMinimalDowntimeResources(createResourceData *ConnectionProfileReq, mutex *sync.Mutex) task.TaskResult[*ConnectionProfileReq]

func (*MockResourceGeneration) RollbackResourceCreation 

func (mrg *MockResourceGeneration) RollbackResourceCreation(ctx context.Context, profiles []*ConnectionProfileReq) error

type MockSchemaFromSource 

type MockSchemaFromSource struct {
	mock.Mock
}

func (*MockSchemaFromSource) SchemaFromDump 

func (msads *MockSchemaFromSource) SchemaFromDump(driver string, spDialect string, ioHelper *utils.IOStreams, processDump ProcessDumpByDialectInterface) (*internal.Conv, error)

type MockValidateOrCreateResources 

type MockValidateOrCreateResources struct {
	mock.Mock
}

func (*MockValidateOrCreateResources) ValidateOrCreateResourcesForShardedMigration 

func (mcr *MockValidateOrCreateResources) ValidateOrCreateResourcesForShardedMigration(ctx context.Context, projectId string, instanceName string, validateOnly bool, region string, sourceProfile profiles.SourceProfile) error

type MockValidateResources 

type MockValidateResources struct {
	mock.Mock
}

func (*MockValidateResources) ValidateResourceGeneration 

func (mvr *MockValidateResources) ValidateResourceGeneration(ctx context.Context, projectId string, instanceId string, sourceProfile profiles.SourceProfile, conv *internal.Conv) error

type PopulateDataConvImpl 

type PopulateDataConvImpl struct{}

type PopulateDataConvInterface 

type PopulateDataConvInterface interface {
	// contains filtered or unexported methods
}

type ProcessDumpByDialectImpl 

type ProcessDumpByDialectImpl struct{}

func (*ProcessDumpByDialectImpl) ProcessDump 

func (pdd *ProcessDumpByDialectImpl) ProcessDump(driver string, conv *internal.Conv, r *internal.Reader) error

ProcessDump invokes process dump function from a sql package based on driver selected.

type ProcessDumpByDialectInterface 

type ProcessDumpByDialectInterface interface {
	ProcessDump(driver string, conv *internal.Conv, r *internal.Reader) error
}

type ReportImpl 

type ReportImpl struct{}

func (*ReportImpl) GenerateReport 

func (r *ReportImpl) GenerateReport(driver string, badWrites map[string]int64, BytesRead int64, banner string, conv *internal.Conv, reportFileName string, dbName string, out *os.File)

Report generates a report of schema and data conversion.

type ReportInterface 

type ReportInterface interface {
	GenerateReport(driver string, badWrites map[string]int64, BytesRead int64, banner string, conv *internal.Conv, reportFileName string, dbName string, out *os.File)
}

type ResourceGenerationImpl 

type ResourceGenerationImpl struct {
	DsAcc         datastream_accessor.DatastreamAccessor
	DsClient      ds.DatastreamClient
	StorageAcc    storageaccessor.StorageAccessor
	StorageClient storageclient.StorageClient
}

func NewResourceGenerationImpl 

func NewResourceGenerationImpl(dsAcc datastream_accessor.DatastreamAccessor, dsClient ds.DatastreamClient, storageAcc storageaccessor.StorageAccessor, storageClient storageclient.StorageClient) *ResourceGenerationImpl

func (ResourceGenerationImpl) GetConnectionProfilesForResources 

func (r ResourceGenerationImpl) GetConnectionProfilesForResources(ctx context.Context, migrationProjectId string, sourceProfile profiles.SourceProfile, region string, validateOnly bool) ([]*ConnectionProfileReq, []*ConnectionProfileReq, error)

Returns source and destination connection profiles to be created

func (ResourceGenerationImpl) PrepareMinimalDowntimeResources 

func (r ResourceGenerationImpl) PrepareMinimalDowntimeResources(createResourceData *ConnectionProfileReq, mutex *sync.Mutex) task.TaskResult[*ConnectionProfileReq]

1. If destination connection profile needs to be created, creates a gcs bucket 2. Creates the connection profile needed for migration

func (ResourceGenerationImpl) RollbackResourceCreation 

func (r ResourceGenerationImpl) RollbackResourceCreation(ctx context.Context, profiles []*ConnectionProfileReq) error

If any of the resource creation fails, deletes all resources that were created

type ResourceGenerationInterface 

type ResourceGenerationInterface interface {
	RollbackResourceCreation(ctx context.Context, profiles []*ConnectionProfileReq) error
	GetConnectionProfilesForResources(ctx context.Context, projectId string, sourceProfile profiles.SourceProfile, region string, validateOnly bool) ([]*ConnectionProfileReq, []*ConnectionProfileReq, error)
	PrepareMinimalDowntimeResources(createResourceData *ConnectionProfileReq, mutex *sync.Mutex) task.TaskResult[*ConnectionProfileReq]
}

type SchemaFromSourceImpl 

type SchemaFromSourceImpl struct{}

func (*SchemaFromSourceImpl) SchemaFromDump 

func (sads *SchemaFromSourceImpl) SchemaFromDump(driver string, spDialect string, ioHelper *utils.IOStreams, processDump ProcessDumpByDialectInterface) (*internal.Conv, error)

type SchemaFromSourceInterface 

type SchemaFromSourceInterface interface {
	SchemaFromDump(driver string, spDialect string, ioHelper *utils.IOStreams, processDump ProcessDumpByDialectInterface) (*internal.Conv, error)
	// contains filtered or unexported methods
}

type SnapshotMigrationImpl 

type SnapshotMigrationImpl struct{}

type SnapshotMigrationInterface 

type SnapshotMigrationInterface interface {
	// contains filtered or unexported methods
}

type ValidateOrCreateResourcesImpl 

type ValidateOrCreateResourcesImpl struct {
	ResourceGenerator ResourceGenerationInterface
	RunParallel       task.RunParallelTasksInterface[*ConnectionProfileReq, *ConnectionProfileReq]
}

func NewValidateOrCreateResourcesImpl 

func NewValidateOrCreateResourcesImpl(dsAcc datastream_accessor.DatastreamAccessor, dsClient ds.DatastreamClient, storageAcc storageaccessor.StorageAccessor, storageClient storageclient.StorageClient) *ValidateOrCreateResourcesImpl

func (*ValidateOrCreateResourcesImpl) ValidateOrCreateResourcesForShardedMigration 

func (c *ValidateOrCreateResourcesImpl) ValidateOrCreateResourcesForShardedMigration(ctx context.Context, migrationProjectId string, instanceName string, validateOnly bool, region string, sourceProfile profiles.SourceProfile) error

1. For each datashard, check if source and destination connection profile exists or not 2. If source connection profile doesn't exists create it or validate if creation is possible. 3. If validation is false and destination connection profile doesn't exists create a corresponding gcs bucket and then a destination connection profile

type ValidateOrCreateResourcesInterface 

type ValidateOrCreateResourcesInterface interface {
	ValidateOrCreateResourcesForShardedMigration(ctx context.Context, migrationProjectId string, instanceName string, validateOnly bool, region string, sourceProfile profiles.SourceProfile) error
}

type ValidateResourcesImpl 

type ValidateResourcesImpl struct {
	SpAcc                     spanneraccessor.SpannerAccessor
	ValidateOrCreateResources ValidateOrCreateResourcesInterface
}

func NewValidateResourcesImpl 

func NewValidateResourcesImpl(spAcc spanneraccessor.SpannerAccessor, dsAcc datastream_accessor.DatastreamAccessor, dsClient ds.DatastreamClient, storageAcc storageaccessor.StorageAccessor, storageClient storageclient.StorageClient) *ValidateResourcesImpl

func (*ValidateResourcesImpl) ValidateResourceGeneration 

func (v *ValidateResourcesImpl) ValidateResourceGeneration(ctx context.Context, projectId string, instanceId string, sourceProfile profiles.SourceProfile, conv *internal.Conv) error

Method to validate if in a minimal downtime migration, required resources can be generated

type ValidateResourcesInterface 

type ValidateResourcesInterface interface {
	ValidateResourceGeneration(ctx context.Context, projectId string, instanceId string, sourceProfile profiles.SourceProfile, conv *internal.Conv) error
}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.