common

README

Retail

The retail module is the base extensible module for all other Go Modules. The Retail module implements the life-cycle and configuration objects required to start and stop all module types.

Life-Cycle

flowchart TB
  i1[Start]
  i2[Load Module]
  i3[Initialize Logging]
  i4[Initialize BigQuery]
  i5[Run]
  i10[Load Loggers]
  i10a[Initialize Logging Client]
  i10b[Logging Ready]
  i11[Load Default]
  i12[Load BQ Logger]
  i13[Load PubSub Logger]
  i20[Load BigQuery]
  i21[Initialize BigQuery Client]
  i22[Initialize Module Tables]
  i30[Error Check]
  i1 --> i2
  i2 --> Logging
  Logging --> BigQuery
  BigQuery --> i30
  i30 --> i5
  subgraph Logging
    direction LR
    i3 --> i10
    i10 --> i10a
    i10a --> i10b
    i10b --> i11
    i10b --> i12
    i10b --> i13
  end
  subgraph BigQuery
   direction LR
   i4 --> i20
   i20 --> i21
   i21 --> i22
  end

Documentation

Index

• Constants
• func ConvertMessageToBigQueryType(t proto.Message) bigquery.ValueSaver
• func QueryableNameFromQualifiedName(qualifiedName string) string
• func SchemaFromFile(fil *os.File) (schema bigquery.Schema, err error)
• type BaseModule
  • func (module *BaseModule) Close()
  • func (module *BaseModule) GetBigQuery() *BigQuery
  • func (module *BaseModule) GetConfig() *Config
  • func (module *BaseModule) GetDefaultLog() *log.Logger
  • func (module *BaseModule) GetDefaultLogger() *log.Logger
  • func (module *BaseModule) GetLogger() *Logger
• type BigQuery
  • func NewBigQuery(ctx context.Context, projectId string, log *log.Logger, datasetName string) (*BigQuery, error)
  • func (bq *BigQuery) CreateTableFromSchema(tableName string, schema bigquery.Schema, table *bigquery.Table, ...)
  • func (bq *BigQuery) GetTable(tableName string) (table *bigquery.Table, err error)
  • func (bq *BigQuery) InitializeTable(tableName string, schemaFileName string, t proto.Message, ...) (out *bigquery.Table, err error)
  • func (bq *BigQuery) Query(sql string) *bigquery.Query
  • func (bq *BigQuery) TableExists(tableName string) (out bool)
• type Config
  • func LoadConfig(fileName string) (moduleConfig *Config, err error)
  • func LoadConfigFromFlags() (*Config, error)
  • func NewConfig(name string, environment string, enabled bool, logLevel string) *Config
• type KeyValuePair
• type Logger
  • func (logger *Logger) GetLogger(name string) (log *logging.Logger)
  • func (logger *Logger) GetStandardLogger(name string) (log *log.Logger)
• type MessageLoaderWrapper
  • func ConvertFromBigQueryToMessage(t proto.Message) *MessageLoaderWrapper
  • func (w *MessageLoaderWrapper) GetMessage() proto.Message
  • func (w *MessageLoaderWrapper) Load(values []bigquery.Value, schema bigquery.Schema) error
• type Module
  • func LoadBaseModule(configFile string) (module Module, err error)
  • func NewBaseModule(config *Config) (module Module, err error)

Constants

const (
	// LogNameFormat is a constant used as "<module_name>_<log_name>_<environment>"
	LogNameFormat  = "%s_%s_%s"
	DBNameFormat   = "%s_%s"
	DefaultLogName = "app"
	DBLogName      = "bq"
)

Constants here define the naming conventions used by the database, logging, and pubsub topics used by the modules. This notion uses the following convention rules from the configuration file: <module_name>_?_environment this convention allows the common to exist across multiple projects and variant environments to support business and testing needs.

const (
	BigQueryDateFormat = "2006-01-02 15:04:05"
)

const (
	FlagConfigFile = "configFile"
)

Functions

func ConvertMessageToBigQueryType

func ConvertMessageToBigQueryType(t proto.Message) bigquery.ValueSaver

ConvertMessageToBigQueryType converts from a message into a message save type

func QueryableNameFromQualifiedName

func QueryableNameFromQualifiedName(qualifiedName string) string

QueryableNameFromQualifiedName - removes the project name with : from the qualified name

func SchemaFromFile

func SchemaFromFile(fil *os.File) (schema bigquery.Schema, err error)

SchemaFromFile reads a JSON file from the operating system this is used during the creation phase of data assets. TODO - Verify this will work with proto bq, it may need to be dropped

Types

type BaseModule

type BaseModule struct {
	// Config - The configuration loaded into the common
	Config *Config
	// BigQuery - The limited BigQuery object, hiding the client and close method
	// to ensure safe operation.
	BigQuery *BigQuery
	// contains filtered or unexported fields
}

BaseModule is the primary structure used for holding state objects.

func (*BaseModule) Close

func (module *BaseModule) Close()

Close - is used to safely close all related objects and instances.

func (*BaseModule) GetBigQuery

func (module *BaseModule) GetBigQuery() *BigQuery

func (*BaseModule) GetConfig

func (module *BaseModule) GetConfig() *Config

func (*BaseModule) GetDefaultLog

func (module *BaseModule) GetDefaultLog() *log.Logger

func (*BaseModule) GetDefaultLogger

func (module *BaseModule) GetDefaultLogger() *log.Logger

GetDefaultLogger - returns a default logger

func (*BaseModule) GetLogger

func (module *BaseModule) GetLogger() *Logger

type BigQuery

type BigQuery struct {
	// contains filtered or unexported fields
}

BigQuery is a state holder for the bigquery.Client, and bigquery.Dataset. This allows for safe concurrent access to the datastore and its tables.

func NewBigQuery

func NewBigQuery(
	ctx context.Context,
	projectId string,
	log *log.Logger,
	datasetName string) (*BigQuery, error)

NewBigQuery is the default constructor for the BigQuery object

func (*BigQuery) CreateTableFromSchema

func (bq *BigQuery) CreateTableFromSchema(
	tableName string,
	schema bigquery.Schema,
	table *bigquery.Table,
	labels map[string]string)

CreateTableFromSchema creates a table from a bq Schema

func (*BigQuery) GetTable

func (bq *BigQuery) GetTable(tableName string) (table *bigquery.Table, err error)

GetTable will get a table from BQ if the table exists, otherwise returns an error

func (*BigQuery) InitializeTable

func (bq *BigQuery) InitializeTable(
	tableName string,
	schemaFileName string,
	t proto.Message,
	additionalTags map[string]string) (out *bigquery.Table, err error)

InitializeTable - reads a table from the provided schema or creates the schema from the struct. Once the schema is created, it is used to create the table if the table does not exist. Once created or read, the table is cached for future use.

func (*BigQuery) Query

func (bq *BigQuery) Query(sql string) *bigquery.Query

func (*BigQuery) TableExists

func (bq *BigQuery) TableExists(tableName string) (out bool)

TableExists determines if the table is already in the dataset

type Config

type Config struct {
	ProjectId   string            `json:"project_id"`
	Name        string            `json:"name"`
	Environment string            `json:"env"`
	Enabled     bool              `json:"enabled"`
	LogLevel    string            `json:"log_level"`
	Labels      map[string]string `json:"labels"`
}

Config is the primary configuration state holder and TO definition.

func LoadConfig

func LoadConfig(fileName string) (moduleConfig *Config, err error)

LoadConfig is used for loading a configuration file from disk.

func LoadConfigFromFlags

func LoadConfigFromFlags() (*Config, error)

func NewConfig

func NewConfig(
	name string,
	environment string,
	enabled bool,
	logLevel string) *Config

NewConfig is the programmatic constructor for the configuration struct.

type KeyValuePair

type KeyValuePair struct {
	Key   string `json:"key"`
	Value string `json:"value"`
}

KeyValuePair is a simple stucture for holding key value pairs

type Logger

type Logger struct {
	// contains filtered or unexported fields
}

Logger is the primary wrapper for holding the logging clinet and map of available / registered loggers and the configured severity level.

func (*Logger) GetLogger

func (logger *Logger) GetLogger(name string) (log *logging.Logger)

GetLogger returns a cloud logger

func (*Logger) GetStandardLogger

func (logger *Logger) GetStandardLogger(name string) (log *log.Logger)

GetStandardLogger returns a standard logger

type MessageLoaderWrapper

type MessageLoaderWrapper struct {
	// contains filtered or unexported fields
}

MessageLoaderWrapper - is a wrapper used to reduce total number of dependencies in other common projects.

func ConvertFromBigQueryToMessage

func ConvertFromBigQueryToMessage(t proto.Message) *MessageLoaderWrapper

ConvertFromBigQueryToMessage converts from the BQ schema into a structured protobuf message type.

func (*MessageLoaderWrapper) GetMessage

func (w *MessageLoaderWrapper) GetMessage() proto.Message

GetMessage is a helper method for converting a wrapped message into a protobuf message

func (*MessageLoaderWrapper) Load

func (w *MessageLoaderWrapper) Load(values []bigquery.Value, schema bigquery.Schema) error

Load is a helper method for loading bq values based on a schema

type Module

type Module interface {
	// GetConfig returns a Configuration object
	GetConfig() *Config
	// GetBigQuery returns a BigQuery wrapper object
	GetBigQuery() *BigQuery
	// GetLogger returns the underlying cloud logger
	GetLogger() *Logger
	// GetDefaultLog returns a native log with a default criticality level.
	GetDefaultLog() *log.Logger
	// Close safely releases all resources
	Close()
}

Module is the basic building block for retail domain modules. Each common represents a clear-cut subdomain model of retail. Please see the default documentation for more details on available modules.

func LoadBaseModule

func LoadBaseModule(configFile string) (module Module, err error)

LoadBaseModule - Reads a JSON configuration from disk and initializes the common. if any errors occur, a non-nil error will be returned. The intention of LoadModule is to be used in conjunction with the Main method of an implementing service.

func NewBaseModule

func NewBaseModule(config *Config) (module Module, err error)

NewBaseModule is a programmatic constructor using a Config object.