e2ehelper

Documentation

Index

• func ColumnWithRawData(column ...string) []string
• type DataFlowTester
  • func NewDataFlowTester(t *testing.T, pluginName string, pluginMeta plugin.PluginMeta) *DataFlowTester
  • func (t *DataFlowTester) CreateSnapshot(dst schema.Tabler, opts TableOptions)
  • func (t *DataFlowTester) ExportRawTable(rawTableName string, csvRelPath string)
  • func (t *DataFlowTester) FlushRawTable(rawTableName string)
  • func (t *DataFlowTester) FlushTabler(dst schema.Tabler)
  • func (t *DataFlowTester) ImportCsvIntoRawTable(csvRelPath string, rawTableName string)
  • func (t *DataFlowTester) ImportCsvIntoTabler(csvRelPath string, dst schema.Tabler)
  • func (t *DataFlowTester) ImportNullableCsvIntoTabler(csvRelPath string, dst schema.Tabler)
  • func (t *DataFlowTester) Subtask(subtaskMeta plugin.SubTaskMeta, taskData interface{})
  • func (t *DataFlowTester) SubtaskContext(taskData interface{}) plugin.SubTaskContext
  • func (t *DataFlowTester) UpdateCSVs(csvDir string, updater func(fileName string, record *pluginhelper.CsvRecord)) errors.Error
  • func (t *DataFlowTester) VerifyTable(dst schema.Tabler, csvRelPath string, targetFields []string)
  • func (t *DataFlowTester) VerifyTableWithOptions(dst schema.Tabler, opts TableOptions)
  • func (t *DataFlowTester) VerifyTableWithRawData(dst schema.Tabler, csvRelPath string, targetFields []string)
• type TableOptions

Examples

• DataFlowTester

Functions

func ColumnWithRawData

func ColumnWithRawData(column ...string) []string

ColumnWithRawData create an Column string with _raw_data_* appending

Types

type DataFlowTester

type DataFlowTester struct {
	Cfg    *viper.Viper
	Db     *gorm.DB
	Dal    dal.Dal
	T      *testing.T
	Name   string
	Plugin plugin.PluginMeta
	Log    log.Logger
}

DataFlowTester use `N`

1. Create a folder under your plugin root folder. i.e. `plugins/gitlab/e2e/ to host all your e2e-tests`
2. Create a folder named `tables` to hold all data in `csv` format
3. Create e2e test-cases to cover all possible data-flow routes

Example code:

See [Gitlab Project Data Flow Test](plugins/gitlab/e2e/project_test.go) for detail

Example

var t *testing.T // stub

var gitlab plugin.PluginMeta
dataflowTester := NewDataFlowTester(t, "gitlab", gitlab)

taskData := &tasks.GitlabTaskData{
	Options: &tasks.GitlabOptions{
		ProjectId: 666888,
	},
}

// import raw data table
dataflowTester.ImportCsvIntoRawTable("./tables/_raw_gitlab_api_issues.csv", "_raw_gitlab_api_issues")

// verify extraction
dataflowTester.FlushTabler(gitlabModels.GitlabProject{})
dataflowTester.Subtask(tasks.ExtractApiIssuesMeta, taskData)
dataflowTester.VerifyTable(
	gitlabModels.GitlabIssue{},
	"tables/_tool_gitlab_issues.csv",
	[]string{
		"gitlab_id",
		"_raw_data_params",
		"_raw_data_table",
		"_raw_data_id",
		"_raw_data_remark",
	},
)

func NewDataFlowTester

func NewDataFlowTester(t *testing.T, pluginName string, pluginMeta plugin.PluginMeta) *DataFlowTester

NewDataFlowTester create a *DataFlowTester to help developer test their subtasks data flow

func (*DataFlowTester) CreateSnapshot

func (t *DataFlowTester) CreateSnapshot(dst schema.Tabler, opts TableOptions)

CreateSnapshot reads rows from database and write them into .csv file.

func (*DataFlowTester) ExportRawTable

func (t *DataFlowTester) ExportRawTable(rawTableName string, csvRelPath string)

ExportRawTable reads rows from raw table and write them into .csv file.

func (*DataFlowTester) FlushRawTable

func (t *DataFlowTester) FlushRawTable(rawTableName string)

FlushRawTable migrate table and deletes all records from specified table

func (*DataFlowTester) FlushTabler

func (t *DataFlowTester) FlushTabler(dst schema.Tabler)

FlushTabler migrate table and deletes all records from specified table

func (*DataFlowTester) ImportCsvIntoRawTable

func (t *DataFlowTester) ImportCsvIntoRawTable(csvRelPath string, rawTableName string)

ImportCsvIntoRawTable imports records from specified csv file into target raw table, note that existing data would be deleted first.

func (*DataFlowTester) ImportCsvIntoTabler

func (t *DataFlowTester) ImportCsvIntoTabler(csvRelPath string, dst schema.Tabler)

ImportCsvIntoTabler imports records from specified csv file into target tabler, the empty string will be taken as NULL. note that existing data would be deleted first.

func (*DataFlowTester) ImportNullableCsvIntoTabler

func (t *DataFlowTester) ImportNullableCsvIntoTabler(csvRelPath string, dst schema.Tabler)

ImportNullableCsvIntoTabler imports records from specified csv file into target tabler, the `NULL` will be taken as NULL. note that existing data would be deleted first.

func (*DataFlowTester) Subtask

func (t *DataFlowTester) Subtask(subtaskMeta plugin.SubTaskMeta, taskData interface{})

Subtask executes specified subtasks

func (*DataFlowTester) SubtaskContext

func (t *DataFlowTester) SubtaskContext(taskData interface{}) plugin.SubTaskContext

SubtaskContext creates a subtask context

func (*DataFlowTester) UpdateCSVs

func (t *DataFlowTester) UpdateCSVs(csvDir string, updater func(fileName string, record *pluginhelper.CsvRecord)) errors.Error

UpdateCSVs scans the supplied directory for .csv files and applies the `updater` function to each. It saves back the result. This is useful when you need to bulk-change a lot of entries across many CSV files in a systematic way.

func (*DataFlowTester) VerifyTable

func (t *DataFlowTester) VerifyTable(dst schema.Tabler, csvRelPath string, targetFields []string)

VerifyTable reads rows from csv file and compare with records from database one by one. You must specify the Primary Key Fields with `pkFields` so DataFlowTester could select the exact record from database, as well as which fields to compare with by specifying `targetFields` parameter. Leaving `targetFields` empty/nil will compare all fields.

func (*DataFlowTester) VerifyTableWithOptions

func (t *DataFlowTester) VerifyTableWithOptions(dst schema.Tabler, opts TableOptions)

VerifyTableWithOptions extends VerifyTable and allows for more advanced usages using TableOptions

func (*DataFlowTester) VerifyTableWithRawData

func (t *DataFlowTester) VerifyTableWithRawData(dst schema.Tabler, csvRelPath string, targetFields []string)

VerifyTableWithRawData use VerifyTable and append the _raw_data_* checking after targetFields

type TableOptions

type TableOptions struct {
	// CSVRelPath relative path to the CSV file that contains the seeded data
	CSVRelPath string
	// TargetFields the fields (columns) to consider for verification. Leave empty to default to all.
	TargetFields []string
	// IgnoreFields the fields (columns) to ignore/skip.
	IgnoreFields []string
	// IgnoreTypes similar to IgnoreFields, this will ignore the fields contained in the type. Useful for ignoring embedded
	// types and their fields in the target model
	IgnoreTypes []interface{}
	// if Nullable is set to be true, only the string `NULL` will be taken as NULL
	Nullable bool
}

TableOptions FIXME ...