seeder

Documentation

Overview

Package seeder main function is to generate sqlite3 databases for use with the api.

It uses schemas from sqlc to create the tagles within the database and will either seed those tables with data from a series of existing files or by generating new|fake data.

Seed is the main function to call and will create the database and insert data.

Used by:

• ./commands/seed (which in turn is called within Dockerfile)
• various test files to create dummy data to test against the api

Index

• Variables
• func DB(dbFile string) (db *sql.DB, err error)
• func InsertFile(ctx context.Context, file string, insertFunc insertF, db *sql.DB) (err error, ts *time.Time)
• func SchemaLoad(db *sql.DB, schemaFile string) (err error)
• func Seed(ctx context.Context, dbF string, schemaF string, dataF string, table string, ...) (db *sql.DB, err error)

Variables

var GENERATOR_FUNCTIONS map[string]generatorF = map[string]generatorF{

	"github_standards": func(ctx context.Context, num int, db *sql.DB) (err error) {
		mu := &sync.Mutex{}
		wg := sync.WaitGroup{}

		owner := fake.String(12)
		tx, err := db.Begin()
		if err != nil {
			return
		}
		defer tx.Rollback()
		q := ghs.New(db)
		qtx := q.WithTx(tx)

		slog.Info("starting generation", slog.String("for", "github_standards"))
		tick := testhelpers.T()
		for x := 0; x < num; x++ {
			wg.Add(1)
			go func(i int) {
				mu.Lock()
				g := ghs.Fake(nil, &owner)
				qtx.Insert(ctx, g.Insertable())
				mu.Unlock()
				wg.Done()
			}(x)
		}

		wg.Wait()
		slog.Info("generation complete", slog.Int("n", num), slog.String("seconds", tick.Stop().Seconds()), slog.String("for", "github_standards"))

		return tx.Commit()
	},

	"aws_costs": func(ctx context.Context, num int, db *sql.DB) (err error) {
		mu := &sync.Mutex{}
		wg := sync.WaitGroup{}

		tx, err := db.Begin()
		if err != nil {
			return
		}
		defer tx.Rollback()
		q := awsc.New(db)
		qtx := q.WithTx(tx)

		slog.Info("starting generation", slog.String("for", "aws_costs"))
		tick := testhelpers.T()
		for x := 0; x < num; x++ {
			wg.Add(1)
			go func(i int) {
				mu.Lock()
				c := awsc.Fake()
				qtx.Insert(ctx, c.Insertable())
				mu.Unlock()
				wg.Done()
			}(x)
		}

		wg.Wait()
		slog.Info("generation complete", slog.Int("n", num), slog.String("seconds", tick.Stop().Seconds()), slog.String("for", "aws_costs"))

		return tx.Commit()
	},
}

map of funcs that inset data from files

var INSERT_FUNCTIONS map[string]insertF = map[string]insertF{
	"github_standards": func(ctx context.Context, fileContent []byte, db *sql.DB) (err error) {

		which := "github_standards"
		repos := []*ghs.GithubStandard{}
		repos, err = convert.Unmarshals[*ghs.GithubStandard](fileContent)
		if err != nil {
			return
		}

		mu := &sync.Mutex{}
		wg := sync.WaitGroup{}
		tx, err := db.Begin()
		if err != nil {
			return
		}
		defer tx.Rollback()
		q := ghs.New(db)
		qtx := q.WithTx(tx)

		slog.Debug("inserting content", slog.String("for", which), slog.Int("count", len(repos)))
		tick := testhelpers.T()
		for _, item := range repos {
			wg.Add(1)
			go func(g *ghs.GithubStandard) {
				mu.Lock()
				qtx.Insert(ctx, g.Insertable())
				mu.Unlock()
				wg.Done()
			}(item)
		}

		wg.Wait()
		slog.Debug("insert complete",
			slog.Int("count", len(repos)),
			slog.String("seconds", tick.Stop().Seconds()),
			slog.String("for", which))

		return tx.Commit()
	},

	"aws_costs": func(ctx context.Context, fileContent []byte, db *sql.DB) (err error) {

		which := "aws_costs"
		items := []*awsc.AwsCost{}
		items, err = convert.Unmarshals[*awsc.AwsCost](fileContent)
		if err != nil {
			return
		}

		mu := &sync.Mutex{}
		wg := sync.WaitGroup{}
		tx, err := db.Begin()
		if err != nil {
			return
		}
		defer tx.Rollback()
		q := awsc.New(db)
		qtx := q.WithTx(tx)

		slog.Debug("inserting content", slog.String("for", which), slog.Int("count", len(items)))
		tick := testhelpers.T()
		for _, item := range items {
			wg.Add(1)
			go func(g *awsc.AwsCost) {
				mu.Lock()
				qtx.Insert(ctx, g.Insertable())
				mu.Unlock()
				wg.Done()
			}(item)
		}

		wg.Wait()
		slog.Debug("insert complete",
			slog.Int("count", len(items)),
			slog.String("seconds", tick.Stop().Seconds()),
			slog.String("for", which))

		return tx.Commit()
	},
}

var TRACKER_FUNCTIONS map[string]trackerF = map[string]trackerF{
	"github_standards": func(ctx context.Context, ts time.Time, db *sql.DB) (err error) {
		q := ghs.New(db)
		err = q.Track(ctx, ts.Format(dates.Format))
		return
	},
	"aws_costs": func(ctx context.Context, ts time.Time, db *sql.DB) (err error) {
		q := awsc.New(db)
		err = q.Track(ctx, ts.Format(dates.Format))
		return
	},
}

Functions

func DB

func DB(dbFile string) (db *sql.DB, err error)

create a db

func InsertFile

func InsertFile(ctx context.Context, file string, insertFunc insertF, db *sql.DB) (err error, ts *time.Time)

InsertFile checks the file exists and reads its content into the insertFunc along with the db pointer.

Small wrapper that also find the modification time of the file. This time is then used to track the age of the data being used

func SchemaLoad

func SchemaLoad(db *sql.DB, schemaFile string) (err error)

SchemaLoad generates a schema from a sqlc file whose filepath is passed

func Seed

func Seed(ctx context.Context, dbF string, schemaF string, dataF string, table string, N int) (db *sql.DB, err error)

Seed tries to create a database at the filepath set (`dbF`) and return the db pointer. This pointer is *NOT* closed, it will need to be handled outside of this function

If there is already a file at `dbF` location, this will exit without error, but will *NOT* create a new version.

A database connection is created (and returned) using the consts.SQL_CONNECTION_PARAMS. The schema file (`schemaF`) is then read and passed into the database to execute and generate the empty tables and indexes. This schema is part of this projects use of sqlc

An insert and tracker function are then checked for, comparing the table name (`table`) against the known list. These are required to be used, so the function will error if a match is not found. See generators.go, insertors.go and trackers.go for list of current versions.

The file pattern (`dataF`) is then used with Glob to find matching files. If they are found, they are iterated over and their contents passed into the insertor function - this function will handle marshaling / coversion and the db record creation

If no files are found but there is a generator function, this will be called instead to place dummy data into the database. The amount of records created is controlled by the `N` parameter