reports

package
v1.0.1-0...-b7ad348 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Dec 7, 2024 License: Apache-2.0 Imports: 10 Imported by: 0

Documentation

Overview

Implements structured report generation for Spanner migration tool.

Index

Constants

This section is empty.

Variables

View Source 
var IssueDB = map[internal.SchemaIssue]struct {
	Brief               string // Short description of issue.
	Severity            Severity
	batch               bool   // Whether multiple instances of this issue are combined.
	Category            string // Standarized issue type
	CategoryDescription string
}{
	internal.DefaultValue:          {Brief: "Some columns have default values which Spanner migration tool does not migrate. Please add the default constraints manually after the migration is complete", Severity: note, Category: "MISSING_DEFAULT_VALUE_CONSTRAINTS", /* contains filtered or unexported fields */},
	internal.ForeignKey:            {Brief: "Spanner does not support foreign keys", Severity: warning, Category: "FOREIGN_KEY_USES"},
	internal.MultiDimensionalArray: {Brief: "Spanner doesn't support multi-dimensional arrays", Severity: warning, Category: "MULTI_DIMENSIONAL_ARRAY_USES"},
	internal.NoGoodType: {Brief: "No appropriate Spanner type. The column will be made nullable in Spanner", Severity: warning, Category: "INAPPROPRIATE_TYPE",
		CategoryDescription: "No appropriate Spanner type"},
	internal.Numeric:              {Brief: "Spanner does not support numeric. This type mapping could lose precision and is not recommended for production use", Severity: warning, Category: "NUMERIC_USES"},
	internal.NumericThatFits:      {Brief: "Spanner does not support numeric, but this type mapping preserves the numeric's specified precision", Severity: suggestion, Category: "NUMERIC_THAT_FITS"},
	internal.Decimal:              {Brief: "Spanner does not support decimal. This type mapping could lose precision and is not recommended for production use", Severity: warning, Category: "DECIMAL_USES"},
	internal.DecimalThatFits:      {Brief: "Spanner does not support decimal, but this type mapping preserves the decimal's specified precision", Severity: suggestion, Category: "DECIMAL_THAT_FITS"},
	internal.Serial:               {Brief: "Spanner does not support autoincrementing types", Severity: warning, Category: "AUTOINCREMENTING_TYPE_USES"},
	internal.AutoIncrement:        {Brief: "Spanner does not support auto_increment attribute", Severity: warning, Category: "AUTO_INCREMENT_ATTRIBUTE_USES"},
	internal.Timestamp:            {Brief: "Spanner timestamp is closer to PostgreSQL timestamptz", Severity: suggestion, Category: "TIMESTAMP_SUGGESTION", /* contains filtered or unexported fields */},
	internal.Datetime:             {Brief: "Spanner timestamp is closer to MySQL timestamp", Severity: warning, Category: "TIMESTAMP_WARNING", /* contains filtered or unexported fields */},
	internal.Time:                 {Brief: "Spanner does not support time/year types", Severity: warning, Category: "TIME_YEAR_TYPE_USES", /* contains filtered or unexported fields */},
	internal.Widened:              {Brief: "Some columns will consume more storage in Spanner", Severity: warning, Category: "STORAGE_WARNING", /* contains filtered or unexported fields */},
	internal.StringOverflow:       {Brief: "String overflow issue might occur as maximum supported length in Spanner is 2621440", Severity: warning, Category: "STRING_OVERFLOW_WARNING"},
	internal.HotspotTimestamp:     {Brief: "Timestamp Hotspot Occured", Severity: warning, Category: "TIMESTAMP_HOTSPOT"},
	internal.HotspotAutoIncrement: {Brief: "Autoincrement Hotspot Occured", Severity: warning, Category: "AUTOINCREMENT_HOTSPOT"},
	internal.InterleavedOrder: {Brief: "can be converted as Interleaved with Table", Severity: suggestion, Category: "INTERLEAVE_TABLE_SUGGESTION",
		CategoryDescription: "Some tables can be interleaved"},
	internal.RedundantIndex:     {Brief: "Redundant Index", Severity: warning, Category: "REDUNDANT_INDEX"},
	internal.AutoIncrementIndex: {Brief: "Auto increment column in Index can create a Hotspot", Severity: warning, Category: "AUTO-INCREMENT_INDEX"},
	internal.InterleaveIndex: {Brief: "can be converted to an Interleave Index", Severity: suggestion, Category: "INTERLEAVE_INDEX_SUGGESTION",
		CategoryDescription: "Some columns can be converted to interleave index"},
	internal.InterleavedNotInOrder: {Brief: "if primary key order parameter is changed for the table", Severity: suggestion, Category: "INTERLEAVED_NOT_IN_ORDER",
		CategoryDescription: "Some tables can be interleaved with parent table if primary key order parameter is changed to 1"},
	internal.InterleavedAddColumn: {Brief: "Candidate for Interleaved Table", Severity: suggestion, Category: "ADD_INTERLEAVED_COLUMN",
		CategoryDescription: "If there is some primary key added in table, it can be interleaved"},
	internal.IllegalName: {Brief: "Names must adhere to the spanner regular expression {a-z|A-Z}[{a-z|A-Z|0-9|_}+]", Severity: warning, Category: "ILLEGAL_NAME"},
	internal.InterleavedRenameColumn: {Brief: "Candidate for Interleaved Table", Severity: suggestion, Category: "RENAME_INTERLEAVED_COLUMN_PRIMARY_KEY",
		CategoryDescription: "If primary key is renamed in table to match the foreign key, the table can be interleaved"},
	internal.InterleavedChangeColumnSize: {Brief: "Candidate for Interleaved Table", Severity: suggestion, Category: "CHANGE_INTERLEAVED_COLUMN_SIZE",
		CategoryDescription: "If column size of this table's primary key is changed to match the foreign key, the table can be interleaved"},
	internal.RowLimitExceeded: {Brief: "Non key columns exceed the spanner limit of 1600 MB. Please modify the column sizes", Severity: Errors, Category: "ROW_LIMIT_EXCEEDED"},
	internal.ShardIdColumnAdded: {Brief: "column was added because this is a sharded migration and this column cannot be dropped", Severity: note, Category: "SHARD_ID_COLUMN_ADDED",
		CategoryDescription: "Shard id column was added because this is a sharded migration and that column couldn't be dropped"},
	internal.ShardIdColumnPrimaryKey: {Brief: "column is not a part of primary key. You may go to the Primary Key tab and add this column as a part of Primary Key", Severity: suggestion, Category: "SHARD_ID_ADD_COLUMN_PRIMARY_KEY",
		CategoryDescription: "Shard id column is not a part of primary key. Please add it to primary key"},
	internal.MissingPrimaryKey: {Category: "MISSING_PRIMARY_KEY",
		CategoryDescription: "Primary Key is missing, synthetic column created as a primary key"},
	internal.UniqueIndexPrimaryKey: {Category: "UNIQUE_INDEX_PRIMARY_KEY",
		CategoryDescription: "Primary Key is missing, unique column(s) used as primary key"},
	internal.ArrayTypeNotSupported:        {Brief: "Array datatype migration is not fully supported. Please validate data after data migration", Severity: warning, Category: "ARRAY_TYPE_NOT_SUPPORTED"},
	internal.SequenceCreated:              {Brief: "Auto Increment has been converted to Sequence, set Skipped Range or Start with Counter to avoid duplicate value errors", Severity: warning, Category: "SEQUENCE_CREATED"},
	internal.ForeignKeyOnDelete:           {Brief: "Spanner supports only ON DELETE CASCADE/NO ACTION", Severity: warning, Category: "FOREIGN_KEY_ACTIONS"},
	internal.ForeignKeyOnUpdate:           {Brief: "Spanner supports only ON UPDATE NO ACTION", Severity: warning, Category: "FOREIGN_KEY_ACTIONS"},
	internal.ForeignKeyActionNotSupported: {Brief: "Spanner supports foreign key action migration only for MySQL and PostgreSQL", Severity: warning, Category: "FOREIGN_KEY_ACTIONS"},
	internal.NumericPKNotSupported:        {Brief: "Spanner PostgreSQL does not support numeric primary keys / unique indices", Severity: warning, Category: "NUMERIC_PK_NOT_SUPPORTED"},
}

IssueDB provides a description and severity for each schema issue. Note on batch: for some issues, we'd like to report just the first instance in a table and suppress other instances i.e. adding more instances of the issue in the same table has little value and could be very noisy. This is controlled via 'batch': if true, we count only the first instance for assessing warnings, and we give only the first instance in the report. TODO: add links in these descriptions to further documentation e.g. for timestamp description.

Functions

func AnalyzeCols 

func AnalyzeCols(conv *internal.Conv, tableId string) (map[string][]internal.SchemaIssue, int64, int64)

AnalyzeCols returns information about the quality of schema mappings for table 'srcTable'. It assumes 'srcTable' is in the conv.SrcSchema map.

func AnalyzeTables 

func AnalyzeTables(conv *internal.Conv, badWrites map[string]int64) (r []tableReport)

AnalyzeTables generates table reports for all processed tables.

func Contains 

func Contains(l []Issue, str string) bool

Contains check string present in list.

func GenerateSummary 

func GenerateSummary(conv *internal.Conv, r []tableReport, badWrites map[string]int64) (string, string)

GenerateSummary creates a summarized version of a tableReport.

func RateSchema 

func RateSchema(cols, warnings, errors int64, missingPKey, summary bool) (string, string)

Types

type ConversionMetadata 

type ConversionMetadata struct {
	ConversionType string        `json:"conversionType"`
	Duration       time.Duration `json:"duration"`
}

type DataReport 

type DataReport struct {
	Rating    string `json:"rating"`
	BadRows   int64  `json:"badRows"`
	TotalRows int64  `json:"totalRows"`
	DryRun    bool   `json:"dryRun"`
}

type IgnoredStatement 

type IgnoredStatement struct {
	StatementType string `json:"statementType"`
	Statement     string `json:"statement"`
}

type Issue 

type Issue struct {
	Category    string `json:"category"`
	Description string `json:"description"`
}

type Issues 

type Issues struct {
	IssueType string  `json:"issueType"`
	IssueList []Issue `json:"issueList"`
}

type NameChange 

type NameChange struct {
	NameChangeType string `json:"nameChangeType"`
	SourceTable    string `json:"sourceTable"`
	OldName        string `json:"oldName"`
	NewName        string `json:"newName"`
}

type ReportImpl 

type ReportImpl struct{}

func (*ReportImpl) GenerateStructuredReport 

func (r *ReportImpl) GenerateStructuredReport(driverName string, dbName string, conv *internal.Conv, badWrites map[string]int64, printTableReports bool, printUnexpecteds bool) StructuredReport

A report consists of the following parts: 1. Summary (overall quality of conversion) 2. Sharding information 2. Ignored statements 3. Conversion duration 4. Migration Type 5. Statement stats (in case of dumps) 6. Name changes 7. Individual table reports (Detailed + Quality of conversion for each) 8. Unexpected conditions

This method the RAW structured report in JSON format. Several utilities can be built on top of this raw, nested JSON data to output the reports in different user and machine friendly formats such as CSV, TXT etc.

func (*ReportImpl) GenerateTextReport 

func (r *ReportImpl) GenerateTextReport(structuredReport StructuredReport, w *bufio.Writer)

report_text.go contains the logic to convert a structured spanner migration tool report to a human readable text report. The structure of the report created is present in (internal/reports/REPORT.md) A sample report can be found in (test_data/mysql_text_report.txt)

type ReportInterface 

type ReportInterface interface {
	GenerateStructuredReport(driverName string, dbName string, conv *internal.Conv, badWrites map[string]int64, printTableReports bool, printUnexpecteds bool) StructuredReport
	GenerateTextReport(structuredReport StructuredReport, w *bufio.Writer)
}

type SchemaReport 

type SchemaReport struct {
	Rating       string `json:"rating"`
	PkMissing    bool   `json:"pkMissing"`
	Issues       int64  `json:"issues"`
	Warnings     int64  `json:"warnings"`
	TotalColumns int64  `json:"totalColumns"`
}

type Severity 

type Severity int
const (
	Errors Severity
)

type StatementStat 

type StatementStat struct {
	Statement  string `json:"statement"`
	Schema     int64  `json:"schema"`
	Data       int64  `json:"data"`
	Skip       int64  `json:"skip"`
	Error      int64  `json:"error"`
	TotalCount int64  `json:"totalCount"`
}

type StatementStats 

type StatementStats struct {
	DriverName     string          `json:"driverName"`
	StatementStats []StatementStat `json:"statementStats"`
}

type StructuredReport 

type StructuredReport struct {
	Summary              Summary              `json:"summary"`
	IsSharded            bool                 `json:"isSharded"`
	IgnoredStatements    []IgnoredStatement   `json:"ignoredStatements"`
	ConversionMetadata   []ConversionMetadata `json:"conversionMetadata"`
	MigrationType        string               `json:"migrationType"`
	StatementStats       StatementStats       `json:"statementStats"`
	NameChanges          []NameChange         `json:"nameChanges"`
	TableReports         []TableReport        `json:"tableReports"`
	UnexpectedConditions UnexpectedConditions `json:"unexpectedConditions"`
	SchemaOnly           bool                 `json:"-"`
}

type Summary 

type Summary struct {
	Text   string `json:"text"`
	Rating string `json:"rating"`
	DbName string `json:"dbName"`
}

type TableReport 

type TableReport struct {
	SrcTableName string       `json:"srcTableName"`
	SpTableName  string       `json:"spTableName"`
	SchemaReport SchemaReport `json:"schemaReport"`
	DataReport   DataReport   `json:"dataReport"`
	Issues       []Issues     `json:"issues"`
}

type UnexpectedCondition 

type UnexpectedCondition struct {
	Count     int64  `json:"count"`
	Condition string `json:"condition"`
}

type UnexpectedConditions 

type UnexpectedConditions struct {
	Reparsed             int64
	UnexpectedConditions []UnexpectedCondition `json:"unexpectedConditions"`
}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.