common

Documentation

Overview

Package common contains common functions and variables used by various scripts and services

Index

• Constants
• Variables
• func GetCSV(filename string) (rows [][]string, err error)
• func GetEnv(key, defaultValue string) string
• func GetLogger(debug, prod bool) *zap.SugaredLogger
• func GetMemUsageMb() uint64
• func HumanBytes(n uint64) string
• func Int64DiffPercentFmt(a, b int64) string
• func IntDiffPercentFmt(a, b int) string
• func LoadSourceLogFiles(log *zap.SugaredLogger, files []string) (txs map[string]map[string]int64, cntProcessedRecords int64)
• func LoadTransactionCSVFiles(log *zap.SugaredLogger, files, knownTxsFiles []string) (txs map[string]*TxSummaryEntry, err error)
• func LoadTxHashesFromMetadataCSVFiles(log *zap.SugaredLogger, files []string) (txs map[string]bool, err error)
• func MustBeFile(log *zap.SugaredLogger, fn string)
• func MustNotExist(log *zap.SugaredLogger, fn string)
• func PrintMemUsage()
• func RLPDecode(rlpBytes []byte) (*types.Transaction, error)
• func RLPStringToTx(rlpHex string) (*types.Transaction, error)
• func SourceAliasesFromEnv() map[string]string
• func TxSourcName(uri string) string
• func TxToRLPString(tx *types.Transaction) (string, error)
• type BlxRawTxMsg
• type EdenRawTxMsg
• type SourceComp
  • func NewSourceComps(args []string) (srcComp []SourceComp)
• type TxSummaryEntry
  • func ParseTx(timestampMs int64, rawTxHex string) (TxSummaryEntry, *types.Transaction, error)
  • func (t TxSummaryEntry) RawTxHex() string
  • func (t TxSummaryEntry) ToCSVRow() []string

Constants

const (
	LocalTag      = "local"
	BloxrouteTag  = "bloxroute"
	ChainboundTag = "chainbound"
)

Variables

var (
	Printer                  = message.NewPrinter(language.English)
	ErrUnsupportedFileFormat = errors.New("unsupported file format")
)

var TxSummaryEntryCSVHeader = []string{
	"timestamp_ms",
	"hash",
	"chain_id",
	"from",
	"to",
	"value",
	"nonce",
	"gas",
	"gas_price",
	"gas_tip_cap",
	"gas_fee_cap",
	"data_size",
	"data_4bytes",
}

Functions

func GetCSV

func GetCSV(filename string) (rows [][]string, err error)

GetCSV returns a CSV content from a file (.csv or .csv.zip)

func GetEnv

func GetEnv(key, defaultValue string) string

func GetLogger

func GetLogger(debug, prod bool) *zap.SugaredLogger

func GetMemUsageMb

func GetMemUsageMb() uint64

func HumanBytes

func HumanBytes(n uint64) string

HumanBytes returns size in the same format as AWS S3

func Int64DiffPercentFmt

func Int64DiffPercentFmt(a, b int64) string

func IntDiffPercentFmt

func IntDiffPercentFmt(a, b int) string

func LoadSourceLogFiles

func LoadSourceLogFiles(log *zap.SugaredLogger, files []string) (txs map[string]map[string]int64, cntProcessedRecords int64)

LoadSourceLogFiles loads sourcelog .csv (or .csv.zip) files (format: <timestamp_ms>,<tx_hash>,<source>) and returns a map[hash][source] = timestampMs

func LoadTransactionCSVFiles

func LoadTransactionCSVFiles(log *zap.SugaredLogger, files, knownTxsFiles []string) (txs map[string]*TxSummaryEntry, err error)

LoadTransactionCSVFiles loads transaction CSV files into a map[txHash]*TxEnvelope All transactions occurring in []knownTxsFiles are skipped

func LoadTxHashesFromMetadataCSVFiles

func LoadTxHashesFromMetadataCSVFiles(log *zap.SugaredLogger, files []string) (txs map[string]bool, err error)

LoadTxHashesFromMetadataCSVFiles loads transaction hashes from metadata CSV (or .csv.zip) files into a map[txHash]bool

func MustBeFile

func MustBeFile(log *zap.SugaredLogger, fn string)

func MustNotExist

func MustNotExist(log *zap.SugaredLogger, fn string)

func PrintMemUsage

func PrintMemUsage()

func RLPDecode

func RLPDecode(rlpBytes []byte) (*types.Transaction, error)

func RLPStringToTx

func RLPStringToTx(rlpHex string) (*types.Transaction, error)

func SourceAliasesFromEnv

func SourceAliasesFromEnv() map[string]string

func TxSourcName

func TxSourcName(uri string) string

func TxToRLPString

func TxToRLPString(tx *types.Transaction) (string, error)

Types

type BlxRawTxMsg

type BlxRawTxMsg struct {
	Params struct {
		Result struct {
			RawTx string
		}
	}
}

type EdenRawTxMsg

type EdenRawTxMsg struct {
	Params struct {
		Result struct {
			RLP string
		}
	}
}

type SourceComp

type SourceComp struct {
	Source    string
	Reference string
}

func NewSourceComps

func NewSourceComps(args []string) (srcComp []SourceComp)

type TxSummaryEntry

type TxSummaryEntry struct {
	Timestamp int64  `parquet:"name=timestamp, type=INT64, convertedtype=TIMESTAMP_MILLIS"`
	Hash      string `parquet:"name=hash, type=BYTE_ARRAY, convertedtype=UTF8, encoding=PLAIN, omitstats=true"`

	ChainID string `parquet:"name=chainId, type=BYTE_ARRAY, convertedtype=UTF8, encoding=PLAIN"`
	From    string `parquet:"name=from, type=BYTE_ARRAY, convertedtype=UTF8, encoding=PLAIN, omitstats=true"`
	To      string `parquet:"name=to, type=BYTE_ARRAY, convertedtype=UTF8, encoding=PLAIN_DICTIONARY"`
	Value   string `parquet:"name=value, type=BYTE_ARRAY, convertedtype=UTF8, encoding=PLAIN, omitstats=true"`
	Nonce   string `parquet:"name=nonce, type=BYTE_ARRAY, convertedtype=UTF8, encoding=PLAIN, omitstats=true"`

	Gas       string `parquet:"name=gas, type=BYTE_ARRAY, convertedtype=UTF8, encoding=PLAIN, omitstats=true"`
	GasPrice  string `parquet:"name=gasPrice, type=BYTE_ARRAY, convertedtype=UTF8, encoding=PLAIN, omitstats=true"`
	GasTipCap string `parquet:"name=gasTipCap, type=BYTE_ARRAY, convertedtype=UTF8, encoding=PLAIN, omitstats=true"`
	GasFeeCap string `parquet:"name=gasFeeCap, type=BYTE_ARRAY, convertedtype=UTF8, encoding=PLAIN, omitstats=true"`

	DataSize   int64  `parquet:"name=dataSize, type=INT64"`
	Data4Bytes string `parquet:"name=data4Bytes, type=BYTE_ARRAY, convertedtype=UTF8, encoding=PLAIN_DICTIONARY"`

	RawTx string `parquet:"name=rawTx, type=BYTE_ARRAY, encoding=PLAIN, omitstats=true"`
}

TxSummaryEntry is a struct that represents a single transaction in the summary CSV and Parquet file see also https://github.com/xitongsys/parquet-go for more details on parquet tags

func ParseTx

func ParseTx(timestampMs int64, rawTxHex string) (TxSummaryEntry, *types.Transaction, error)

func (TxSummaryEntry) RawTxHex

func (t TxSummaryEntry) RawTxHex() string

func (TxSummaryEntry) ToCSVRow

func (t TxSummaryEntry) ToCSVRow() []string