reports

package
v1.0.1-0...-386defc Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Oct 18, 2023 License: Apache-2.0 Imports: 10 Imported by: 0

Documentation

Overview

Implements structured report generation for Spanner migration tool.

Index

Constants

This section is empty.

Variables

View Source 
var IssueDB = map[internal.SchemaIssue]struct {
	Brief               string // Short description of issue.
	severity            severity
	batch               bool   // Whether multiple instances of this issue are combined.
	Category            string // Standarized issue type
	CategoryDescription string
}{
	internal.DefaultValue:          {Brief: "Some columns have default values which Spanner migration tool does not migrate. Please add the default constraints manually after the migration is complete", Category: "MISSING_DEFAULT_VALUE_CONSTRAINTS", /* contains filtered or unexported fields */},
	internal.ForeignKey:            {Brief: "Spanner does not support foreign keys", Category: "FOREIGN_KEY_USES", /* contains filtered or unexported fields */},
	internal.MultiDimensionalArray: {Brief: "Spanner doesn't support multi-dimensional arrays", Category: "MULTI_DIMENSIONAL_ARRAY_USES", /* contains filtered or unexported fields */},
	internal.NoGoodType: {Brief: "No appropriate Spanner type. The column will be made nullable in Spanner", Category: "INAPPROPRIATE_TYPE",
		CategoryDescription: "No appropriate Spanner type",
		// contains filtered or unexported fields
	},
	internal.Numeric:              {Brief: "Spanner does not support numeric. This type mapping could lose precision and is not recommended for production use", Category: "NUMERIC_USES", /* contains filtered or unexported fields */},
	internal.NumericThatFits:      {Brief: "Spanner does not support numeric, but this type mapping preserves the numeric's specified precision", Category: "NUMERIC_THAT_FITS", /* contains filtered or unexported fields */},
	internal.Decimal:              {Brief: "Spanner does not support decimal. This type mapping could lose precision and is not recommended for production use", Category: "DECIMAL_USES", /* contains filtered or unexported fields */},
	internal.DecimalThatFits:      {Brief: "Spanner does not support decimal, but this type mapping preserves the decimal's specified precision", Category: "DECIMAL_THAT_FITS", /* contains filtered or unexported fields */},
	internal.Serial:               {Brief: "Spanner does not support autoincrementing types", Category: "AUTOINCREMENTING_TYPE_USES", /* contains filtered or unexported fields */},
	internal.AutoIncrement:        {Brief: "Spanner does not support auto_increment attribute", Category: "AUTO_INCREMENT_ATTRIBUTE_USES", /* contains filtered or unexported fields */},
	internal.Timestamp:            {Brief: "Spanner timestamp is closer to PostgreSQL timestamptz", Category: "TIMESTAMP_SUGGESTION", /* contains filtered or unexported fields */},
	internal.Datetime:             {Brief: "Spanner timestamp is closer to MySQL timestamp", Category: "TIMESTAMP_WARNING", /* contains filtered or unexported fields */},
	internal.Time:                 {Brief: "Spanner does not support time/year types", Category: "TIME_YEAR_TYPE_USES", /* contains filtered or unexported fields */},
	internal.Widened:              {Brief: "Some columns will consume more storage in Spanner", Category: "STORAGE_WARNING", /* contains filtered or unexported fields */},
	internal.StringOverflow:       {Brief: "String overflow issue might occur as maximum supported length in Spanner is 2621440", Category: "STRING_OVERFLOW_WARNING", /* contains filtered or unexported fields */},
	internal.HotspotTimestamp:     {Brief: "Timestamp Hotspot Occured", Category: "TIMESTAMP_HOTSPOT", /* contains filtered or unexported fields */},
	internal.HotspotAutoIncrement: {Brief: "Autoincrement Hotspot Occured", Category: "AUTOINCREMENT_HOTSPOT", /* contains filtered or unexported fields */},
	internal.InterleavedOrder: {Brief: "can be converted as Interleaved with Table", Category: "INTERLEAVE_TABLE_SUGGESTION",
		CategoryDescription: "Some tables can be interleaved",
		// contains filtered or unexported fields
	},
	internal.RedundantIndex:     {Brief: "Redundant Index", Category: "REDUNDANT_INDEX", /* contains filtered or unexported fields */},
	internal.AutoIncrementIndex: {Brief: "Auto increment column in Index can create a Hotspot", Category: "AUTO-INCREMENT_INDEX", /* contains filtered or unexported fields */},
	internal.InterleaveIndex: {Brief: "can be converted to an Interleave Index", Category: "INTERLEAVE_INDEX_SUGGESTION",
		CategoryDescription: "Some columns can be converted to interleave index",
		// contains filtered or unexported fields
	},
	internal.InterleavedNotInOrder: {Brief: "if primary key order parameter is changed for the table", Category: "INTERLEAVED_NOT_IN_ORDER",
		CategoryDescription: "Some tables can be interleaved with parent table if primary key order parameter is changed to 1",
		// contains filtered or unexported fields
	},
	internal.InterleavedAddColumn: {Brief: "Candidate for Interleaved Table", Category: "ADD_INTERLEAVED_COLUMN",
		CategoryDescription: "If there is some primary key added in table, it can be interleaved",
		// contains filtered or unexported fields
	},
	internal.IllegalName: {Brief: "Names must adhere to the spanner regular expression {a-z|A-Z}[{a-z|A-Z|0-9|_}+]", Category: "ILLEGAL_NAME", /* contains filtered or unexported fields */},
	internal.InterleavedRenameColumn: {Brief: "Candidate for Interleaved Table", Category: "RENAME_INTERLEAVED_COLUMN_PRIMARY_KEY",
		CategoryDescription: "If primary key is renamed in table to match the foreign key, the table can be interleaved",
		// contains filtered or unexported fields
	},
	internal.InterleavedChangeColumnSize: {Brief: "Candidate for Interleaved Table", Category: "CHANGE_INTERLEAVED_COLUMN_SIZE",
		CategoryDescription: "If column size of this table's primary key is changed to match the foreign key, the table can be interleaved",
		// contains filtered or unexported fields
	},
	internal.RowLimitExceeded: {Brief: "Non key columns exceed the spanner limit of 1600 MB. Please modify the column sizes", Category: "ROW_LIMIT_EXCEEDED", /* contains filtered or unexported fields */},
	internal.ShardIdColumnAdded: {Brief: "column was added because this is a sharded migration and this column cannot be dropped", Category: "SHARD_ID_COLUMN_ADDED",
		CategoryDescription: "Shard id column was added because this is a sharded migration and that column couldn't be dropped",
		// contains filtered or unexported fields
	},
	internal.ShardIdColumnPrimaryKey: {Brief: "column is not a part of primary key. You may go to the Primary Key tab and add this column as a part of Primary Key", Category: "SHARD_ID_ADD_COLUMN_PRIMARY_KEY",
		CategoryDescription: "Shard id column is not a part of primary key. Please add it to primary key",
		// contains filtered or unexported fields
	},
	internal.MissingPrimaryKey: {Category: "MISSING_PRIMARY_KEY",
		CategoryDescription: "Primary Key is missing, synthetic column created as a primary key"},
	internal.UniqueIndexPrimaryKey: {Category: "UNIQUE_INDEX_PRIMARY_KEY",
		CategoryDescription: "Primary Key is missing, unique column(s) used as primary key"},
	internal.ArrayTypeNotSupported: {Brief: "Array datatype migration is not fully supported. Please validate data after data migration", Category: "ARRAY_TYPE_NOT_SUPPORTED", /* contains filtered or unexported fields */},
}

IssueDB provides a description and severity for each schema issue. Note on batch: for some issues, we'd like to report just the first instance in a table and suppress other instances i.e. adding more instances of the issue in the same table has little value and could be very noisy. This is controlled via 'batch': if true, we count only the first instance for assessing warnings, and we give only the first instance in the report. TODO: add links in these descriptions to further documentation e.g. for timestamp description.

Functions

func AnalyzeCols 

func AnalyzeCols(conv *internal.Conv, tableId string) (map[string][]internal.SchemaIssue, int64, int64)

AnalyzeCols returns information about the quality of schema mappings for table 'srcTable'. It assumes 'srcTable' is in the conv.SrcSchema map.

func AnalyzeTables 

func AnalyzeTables(conv *internal.Conv, badWrites map[string]int64) (r []tableReport)

AnalyzeTables generates table reports for all processed tables.

func Contains 

func Contains(l []Issue, str string) bool

Contains check string present in list.

func GenerateSummary 

func GenerateSummary(conv *internal.Conv, r []tableReport, badWrites map[string]int64) (string, string)

GenerateSummary creates a summarized version of a tableReport.

func GenerateTextReport 

func GenerateTextReport(structuredReport StructuredReport, w *bufio.Writer)

report_text.go contains the logic to convert a structured spanner migration tool report to a human readable text report. The structure of the report created is present in (internal/reports/REPORT.md) A sample report can be found in (test_data/mysql_text_report.txt)

func RateSchema 

func RateSchema(cols, warnings, errors int64, missingPKey, summary bool) (string, string)

Types

type ConversionMetadata 

type ConversionMetadata struct {
	ConversionType string        `json:"conversionType"`
	Duration       time.Duration `json:"duration"`
}

type DataReport 

type DataReport struct {
	Rating    string `json:"rating"`
	BadRows   int64  `json:"badRows"`
	TotalRows int64  `json:"totalRows"`
	DryRun    bool   `json:"dryRun"`
}

type IgnoredStatement 

type IgnoredStatement struct {
	StatementType string `json:"statementType"`
	Statement     string `json:"statement"`
}

type Issue 

type Issue struct {
	Category    string `json:"category"`
	Description string `json:"description"`
}

type Issues 

type Issues struct {
	IssueType string  `json:"issueType"`
	IssueList []Issue `json:"issueList"`
}

type NameChange 

type NameChange struct {
	NameChangeType string `json:"nameChangeType"`
	SourceTable    string `json:"sourceTable"`
	OldName        string `json:"oldName"`
	NewName        string `json:"newName"`
}

type SchemaReport 

type SchemaReport struct {
	Rating       string `json:"rating"`
	PkMissing    bool   `json:"pkMissing"`
	Issues       int64  `json:"issues"`
	Warnings     int64  `json:"warnings"`
	TotalColumns int64  `json:"totalColumns"`
}

type StatementStat 

type StatementStat struct {
	Statement  string `json:"statement"`
	Schema     int64  `json:"schema"`
	Data       int64  `json:"data"`
	Skip       int64  `json:"skip"`
	Error      int64  `json:"error"`
	TotalCount int64  `json:"totalCount"`
}

type StatementStats 

type StatementStats struct {
	DriverName     string          `json:"driverName"`
	StatementStats []StatementStat `json:"statementStats"`
}

type StructuredReport 

type StructuredReport struct {
	Summary              Summary              `json:"summary"`
	IsSharded            bool                 `json:"isSharded"`
	IgnoredStatements    []IgnoredStatement   `json:"ignoredStatements"`
	ConversionMetadata   []ConversionMetadata `json:"conversionMetadata"`
	MigrationType        string               `json:"migrationType"`
	StatementStats       StatementStats       `json:"statementStats"`
	NameChanges          []NameChange         `json:"nameChanges"`
	TableReports         []TableReport        `json:"tableReports"`
	UnexpectedConditions UnexpectedConditions `json:"unexpectedConditions"`
	SchemaOnly           bool                 `json:"-"`
}

func GenerateStructuredReport 

func GenerateStructuredReport(driverName string, dbName string, conv *internal.Conv, badWrites map[string]int64, printTableReports bool, printUnexpecteds bool) StructuredReport

A report consists of the following parts: 1. Summary (overall quality of conversion) 2. Sharding information 2. Ignored statements 3. Conversion duration 4. Migration Type 5. Statement stats (in case of dumps) 6. Name changes 7. Individual table reports (Detailed + Quality of conversion for each) 8. Unexpected conditions

This method the RAW structured report in JSON format. Several utilities can be built on top of this raw, nested JSON data to output the reports in different user and machine friendly formats such as CSV, TXT etc.

type Summary 

type Summary struct {
	Text   string `json:"text"`
	Rating string `json:"rating"`
	DbName string `json:"dbName"`
}

type TableReport 

type TableReport struct {
	SrcTableName string       `json:"srcTableName"`
	SpTableName  string       `json:"spTableName"`
	SchemaReport SchemaReport `json:"schemaReport"`
	DataReport   DataReport   `json:"dataReport"`
	Issues       []Issues     `json:"issues"`
}

type UnexpectedCondition 

type UnexpectedCondition struct {
	Count     int64  `json:"count"`
	Condition string `json:"condition"`
}

type UnexpectedConditions 

type UnexpectedConditions struct {
	Reparsed             int64
	UnexpectedConditions []UnexpectedCondition `json:"unexpectedConditions"`
}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.