Documentation ¶
Index ¶
- Constants
- func ActorHit(ctx *Ctx, actorName string) bool
- func ActorIDOrNil(actPtr *Actor) interface{}
- func ActorLoginOrNil(actPtr *Actor, maybeHide func(string) string) interface{}
- func AddNIntervals(dt time.Time, n int, ...) time.Time
- func AddTSPoint(ctx *Ctx, pts *TSPoints, pt TSPoint)
- func BoolOrNil(boolPtr *bool) interface{}
- func CleanUTF8(str string) string
- func ClearDBLogs()
- func CommentIDOrNil(commPtr *Comment) interface{}
- func ComputePeriodAtThisDate(ctx *Ctx, period string, dt time.Time) bool
- func CreateDatabaseIfNeeded(ctx *Ctx) bool
- func CreateTable(tdef string) string
- func DatabaseExists(ctx *Ctx, closeConn bool) (exists bool, c *sql.DB)
- func DayStart(dt time.Time) time.Time
- func DescriblePeriodInHours(hrs float64) (desc string)
- func DropDatabaseIfExists(ctx *Ctx) bool
- func EnvReplace(prefix, suffix string) map[string]string
- func EnvRestore(env map[string]string)
- func ExecCommand(ctx *Ctx, cmdAndArgs []string, env map[string]string) (string, error)
- func ExecSQL(con *sql.DB, ctx *Ctx, query string, args ...interface{}) (sql.Result, error)
- func ExecSQLTx(con *sql.Tx, ctx *Ctx, query string, args ...interface{}) (sql.Result, error)
- func ExecSQLTxWithErr(con *sql.Tx, ctx *Ctx, query string, args ...interface{}) sql.Result
- func ExecSQLWithErr(con *sql.DB, ctx *Ctx, query string, args ...interface{}) sql.Result
- func FatalNoLog(err error) string
- func FatalOnError(err error) string
- func Fatalf(f string, a ...interface{})
- func FirstIntOrNil(intPtrs []*int) interface{}
- func ForkeeIDOrNil(forkPtr *Forkee) interface{}
- func ForkeeNameOrNil(forkPtr *Forkee) interface{}
- func ForkeeOldIDOrNil(forkPtr *ForkeeOld) interface{}
- func GHClient(ctx *Ctx) (ghCtx context.Context, client *github.Client)
- func GetHidden(configFile string) map[string]string
- func GetIntervalFunctions(intervalAbbr string, allowUnknown bool) (interval string, n int, ...)
- func GetRateLimits(gctx context.Context, gc *github.Client, core bool) (int, int, time.Duration)
- func GetTagValues(con *sql.DB, ctx *Ctx, name, key string) (ret []string)
- func GetThreadsNum(ctx *Ctx) int
- func HashStrings(strs []string) int
- func HourStart(dt time.Time) time.Time
- func InsertIgnore(query string) string
- func IntOrNil(intPtr *int) interface{}
- func IsProjectDisabled(ctx *Ctx, proj string, yamlDisabled bool) bool
- func IssueIDOrNil(issuePtr *Issue) interface{}
- func MakeOldRepoName(repo *ForkeeOld) string
- func MakeUniqueSort(ary []string) (outAry []string)
- func MaybeHideFunc(shas map[string]string) func(string) string
- func Mgetc(ctx *Ctx) string
- func MilestoneIDOrNil(milPtr *Milestone) interface{}
- func MonthStart(dt time.Time) time.Time
- func NValue(index int) string
- func NValues(n int) string
- func NegatedBoolOrNil(boolPtr *bool) interface{}
- func NextDayStart(dt time.Time) time.Time
- func NextHourStart(dt time.Time) time.Time
- func NextMonthStart(dt time.Time) time.Time
- func NextQuarterStart(dt time.Time) time.Time
- func NextWeekStart(dt time.Time) time.Time
- func NextYearStart(dt time.Time) time.Time
- func NormalizeName(str string) string
- func OrgIDOrNil(orgPtr *Org) interface{}
- func OrgLoginOrNil(orgPtr *Org) interface{}
- func PgConn(ctx *Ctx) *sql.DB
- func PgConnDB(ctx *Ctx, dbName string) *sql.DB
- func PrepareQuickRangeQuery(sql, period, from, to string) string
- func PrettyPrintJSON(jsonBytes []byte) []byte
- func PrevDayStart(dt time.Time) time.Time
- func PrevHourStart(dt time.Time) time.Time
- func PrevMonthStart(dt time.Time) time.Time
- func PrevQuarterStart(dt time.Time) time.Time
- func PrevWeekStart(dt time.Time) time.Time
- func PrevYearStart(dt time.Time) time.Time
- func Printf(format string, args ...interface{}) (n int, err error)
- func ProcessAnnotations(ctx *Ctx, annotations *Annotations, startDate, joinDate *time.Time)
- func ProcessTag(con *sql.DB, ctx *Ctx, tg *Tag, replaces [][]string)
- func ProgressInfo(i, n int, start time.Time, last *time.Time, period time.Duration, msg string)
- func PullRequestIDOrNil(prPtr *PullRequest) interface{}
- func QuarterStart(dt time.Time) time.Time
- func QueryRowSQL(con *sql.DB, ctx *Ctx, query string, args ...interface{}) *sql.Row
- func QueryRowSQLTx(tx *sql.Tx, ctx *Ctx, query string, args ...interface{}) *sql.Row
- func QuerySQL(con *sql.DB, ctx *Ctx, query string, args ...interface{}) (*sql.Rows, error)
- func QuerySQLTx(con *sql.Tx, ctx *Ctx, query string, args ...interface{}) (*sql.Rows, error)
- func QuerySQLTxWithErr(con *sql.Tx, ctx *Ctx, query string, args ...interface{}) *sql.Rows
- func QuerySQLWithErr(con *sql.DB, ctx *Ctx, query string, args ...interface{}) *sql.Rows
- func ReadFile(ctx *Ctx, path string) ([]byte, error)
- func ReleaseIDOrNil(relPtr *Release) interface{}
- func RepoHit(ctx *Ctx, fullName string, forg, frepo map[string]struct{}) bool
- func RepoIDOrNil(repoPtr *Repo) interface{}
- func RepoNameOrNil(repoPtr *Repo) interface{}
- func SkipEmpty(arr []string) []string
- func Slugify(arg string) string
- func StringOrNil(strPtr *string) interface{}
- func StringsMapToArray(f func(string) string, strArr []string) []string
- func StringsMapToSet(f func(string) string, strArr []string) map[string]struct{}
- func StringsSetKeys(set map[string]struct{}) []string
- func StripUnicode(str string) string
- func Structure(ctx *Ctx)
- func TableColumnExists(con *sql.DB, ctx *Ctx, tableName, columnName string) bool
- func TableExists(con *sql.DB, ctx *Ctx, tableName string) bool
- func TimeOrNil(timePtr *time.Time) interface{}
- func TimeParseAny(dtStr string) time.Time
- func ToGHADate(dt time.Time) string
- func ToYMDDate(dt time.Time) string
- func ToYMDHDate(dt time.Time) string
- func ToYMDHMSDate(dt time.Time) string
- func TruncStringOrNil(strPtr *string, maxLen int) interface{}
- func TruncToBytes(str string, size int) string
- func WeekStart(dt time.Time) time.Time
- func WriteTSPoints(ctx *Ctx, con *sql.DB, pts *TSPoints, mergeSeries string, mut *sync.Mutex)
- func YearStart(dt time.Time) time.Time
- type Actor
- type AllProjects
- type Annotation
- type Annotations
- type AnnotationsByDate
- type AnyArray
- type Asset
- type Author
- type Branch
- type Comment
- type Commit
- type Ctx
- type Dummy
- type Event
- type EventOld
- type Forkee
- type ForkeeOld
- type Int64Ary
- type Issue
- type Label
- type Milestone
- type Org
- type Page
- type Payload
- type PayloadOld
- type Project
- type PullRequest
- type Release
- type Repo
- type TSPoint
- type TSPoints
- type Tag
- type Tags
- type Team
Constants ¶
const DataDir string = "/etc/gha2db/"
DataDir - common constant string
const Devstats string = "devstats"
Devstats - common constant string
const EngineIsClosedError string = "engine is closed"
EngineIsClosedError - common constant string
const GHA string = "gha"
GHA - common constant string
const GHAAdmin string = "gha_admin"
GHAAdmin - common constant string
const HideCfgFile string = "hide/hide.csv"
HideCfgFile - common constant string
const LocalGitScripts string = "./git/"
LocalGitScripts - common constant string
const Localhost string = "localhost"
Localhost - common constant string
const Metrics string = "metrics/"
Metrics - common constant string
const Now string = "now"
Now - common constant string
const Null string = "null"
Null - common constant string
const Password string = "password"
Password - common constant string
const Quarter string = "quarter"
Quarter - common constant string
const Retry string = "retry"
Retry - common constant string
const TimeCol string = "time"
TimeCol - common constant string
const TimeoutError string = "{\"error\":\"timeout\"}\n"
TimeoutError - common constant string
const Today string = "today"
Today - common constant string
const Unset string = "{{unset}}"
Unset - common constant string
Variables ¶
This section is empty.
Functions ¶
func ActorIDOrNil ¶
func ActorIDOrNil(actPtr *Actor) interface{}
ActorIDOrNil - return Actor ID from pointer or nil
func ActorLoginOrNil ¶
ActorLoginOrNil - return Actor Login from pointer or nil
func AddNIntervals ¶
func AddNIntervals(dt time.Time, n int, nextIntervalStart, prevIntervalStart func(time.Time) time.Time) time.Time
AddNIntervals adds (using nextIntervalStart) or subtracts (using prevIntervalStart) N itervals to the given date Functions Next/Prev can use Hour, Day, Week, Month, Quarter, Year functions (defined in this module) or other custom defined functions With `func(time.Time) time.Time` signature
func AddTSPoint ¶ added in v0.5.0
AddTSPoint add single point to the batch
func BoolOrNil ¶
func BoolOrNil(boolPtr *bool) interface{}
BoolOrNil - return either nil or value of boolPtr
func ClearDBLogs ¶
func ClearDBLogs()
ClearDBLogs clears logs older by defined period (in context.go) It clears logs on `devstats` database
func CommentIDOrNil ¶
func CommentIDOrNil(commPtr *Comment) interface{}
CommentIDOrNil - return Comment ID from pointer or nil
func ComputePeriodAtThisDate ¶
ComputePeriodAtThisDate - for some longer periods, only recalculate them on specific dates hourly period is always calculated daily period is always calculated multiple days period are calculaded at hours: 1, 5, 9, 13, 17, 21 (UTC) annotation ranges are calculated: from last release to now - every 2 hours for past ranges only once (calculation is marked as computed) at 2 AM weekly ranges are calculated at hours: 0, 4, 8, 12, 16, 20 monthly, quarterly, yearly ranges are calculated at midnight
func CreateDatabaseIfNeeded ¶
CreateDatabaseIfNeeded - creates requested database if not exists Returns true if database was not existing existed and created dropped
func CreateTable ¶
CreateTable is used to replace DB specific parts of Create Table SQL statement
func DatabaseExists ¶
DatabaseExists - checks if database stored in context exists If closeConn is true - then it closes connection after checking if database exists If closeConn is false, then it returns open connection to default database "postgres"
func DescriblePeriodInHours ¶
DescriblePeriodInHours - return string description of a time period given in hours
func DropDatabaseIfExists ¶
DropDatabaseIfExists - drops requested database if exists Returns true if database existed and was dropped
func EnvReplace ¶ added in v0.4.0
EnvReplace - replace all environment variables starting with "prefix" with contents of variables with "suffix" added - if defined If prefix is "DB_" and suffix is "_SRC" then: So if there is "DB_HOST_SRC" variable defined - it will replace "DB_HOST" and so on
func EnvRestore ¶ added in v0.4.0
EnvRestore - restores all environment variables given in the map
func ExecCommand ¶
ExecCommand - execute command given by array of strings with eventual environment map
func ExecSQL ¶
ExecSQL executes given SQL on Postgres DB (and return single state result, that doesn't need to be closed)
func ExecSQLTx ¶
ExecSQLTx executes given SQL on Postgres DB (and return single state result, that doesn't need to be closed) It is for running inside transaction
func ExecSQLTxWithErr ¶
ExecSQLTxWithErr wrapper to ExecSQLTx that exists on error It is for running inside transaction
func ExecSQLWithErr ¶
ExecSQLWithErr wrapper to ExecSQL that exists on error
func FatalNoLog ¶ added in v0.5.0
FatalNoLog displays error message (if error present) and exits program, should be used for very early init state
func FatalOnError ¶
FatalOnError displays error message (if error present) and exits program
func Fatalf ¶ added in v0.3.0
func Fatalf(f string, a ...interface{})
Fatalf - it will call FatalOnError using fmt.Errorf with args provided
func FirstIntOrNil ¶
func FirstIntOrNil(intPtrs []*int) interface{}
FirstIntOrNil - return either nil or value of intPtr
func ForkeeIDOrNil ¶
func ForkeeIDOrNil(forkPtr *Forkee) interface{}
ForkeeIDOrNil - return Forkee ID from pointer or nil
func ForkeeNameOrNil ¶
func ForkeeNameOrNil(forkPtr *Forkee) interface{}
ForkeeNameOrNil - return Forkee Name from pointer or nil
func ForkeeOldIDOrNil ¶
func ForkeeOldIDOrNil(forkPtr *ForkeeOld) interface{}
ForkeeOldIDOrNil - return ForkeeOld ID from pointer or nil
func GetIntervalFunctions ¶
func GetIntervalFunctions(intervalAbbr string, allowUnknown bool) (interval string, n int, intervalStart, nextIntervalStart, prevIntervalStart func(time.Time) time.Time)
GetIntervalFunctions - return interval name, interval number, interval start, next, prev function from interval abbr: h|d2|w3|m4|q|y w3 = 3 weeks, q2 = 2 quarters, y = year (1), d7 = 7 days (not the same as w), m3 = 3 months (not the same as q)
func GetRateLimits ¶ added in v0.5.0
GetRateLimits - returns all and remaining API points and duration to wait for reset when core=true - returns Core limits, when core=false returns Search limits
func GetTagValues ¶
GetTagValues returns tag values for a given key
func GetThreadsNum ¶
GetThreadsNum returns the number of available CPUs If environment variable GHA2DB_ST is set it retuns 1 It can be used to debug single threaded verion It runs on 95% CPU power by default
func HashStrings ¶
HashStrings - returns unique Hash for strings array This value is supposed to be used as ID (negative) to mark it was artificially generated
func InsertIgnore ¶
InsertIgnore - will return insert statement with ignore option specific for DB
func IntOrNil ¶
func IntOrNil(intPtr *int) interface{}
IntOrNil - return either nil or value of intPtr
func IsProjectDisabled ¶
IsProjectDisabled - checks if project is disabled or not: fullName comes from makeOldRepoName for pre-2015 data! yamlDisabled (this is from projects.yaml - can be true or false) it also checks context (which can override `disabled: true` from projects.yaml) +pro1,-pro2 creates map {"pro1":true, "pro2":false}
func IssueIDOrNil ¶
func IssueIDOrNil(issuePtr *Issue) interface{}
IssueIDOrNil - return Issue ID from pointer or nil
func MakeOldRepoName ¶
MakeOldRepoName - before 2015 repository name should be Organization/Name (if Organization present) or just Name
func MakeUniqueSort ¶
MakeUniqueSort - make string array unique & sorted
func MaybeHideFunc ¶ added in v0.6.0
MaybeHideFunc - use closure as a data storage
func MilestoneIDOrNil ¶
func MilestoneIDOrNil(milPtr *Milestone) interface{}
MilestoneIDOrNil - return Milestone ID from pointer or nil
func MonthStart ¶
MonthStart - return time rounded to current month start
func NegatedBoolOrNil ¶
func NegatedBoolOrNil(boolPtr *bool) interface{}
NegatedBoolOrNil - return either nil or negated value of boolPtr
func NextDayStart ¶
NextDayStart - return time rounded to next day start
func NextHourStart ¶
NextHourStart - return time rounded to next hour start
func NextMonthStart ¶
NextMonthStart - return time rounded to next month start
func NextQuarterStart ¶
NextQuarterStart - return time rounded to next quarter start
func NextWeekStart ¶
NextWeekStart - return time rounded to next week start
func NextYearStart ¶
NextYearStart - return time rounded to next year start
func NormalizeName ¶
NormalizeName - clean DB string from -, /, ., " ",), (, ], [ trim leading and trailing space, lowercase Normalize Unicode characters
func OrgIDOrNil ¶
func OrgIDOrNil(orgPtr *Org) interface{}
OrgIDOrNil - return Org ID from pointer or nil
func OrgLoginOrNil ¶
func OrgLoginOrNil(orgPtr *Org) interface{}
OrgLoginOrNil - return Org ID from pointer or nil
func PgConnDB ¶
PgConnDB Connects to Postgres database (with specific DB name) uses database 'dbname' instead of 'PgDB'
func PrepareQuickRangeQuery ¶
PrepareQuickRangeQuery Perpares query using either ready `period` string or using `from` and `to` strings Values to replace are specially encoded {{period:alias.column}} Can either replace with: (alias.column >= now() - 'period'::interval) Or (alias.column >= 'from' and alias.column < 'to')
func PrettyPrintJSON ¶
PrettyPrintJSON - pretty formats raw JSON bytes
func PrevDayStart ¶
PrevDayStart - return time rounded to prev day start
func PrevHourStart ¶
PrevHourStart - return time rounded to prev hour start
func PrevMonthStart ¶
PrevMonthStart - return time rounded to prev month start
func PrevQuarterStart ¶
PrevQuarterStart - return time rounded to prev quarter start
func PrevWeekStart ¶
PrevWeekStart - return time rounded to prev week start
func PrevYearStart ¶
PrevYearStart - return time rounded to prev year start
func ProcessAnnotations ¶
func ProcessAnnotations(ctx *Ctx, annotations *Annotations, startDate, joinDate *time.Time)
ProcessAnnotations Creates IfluxDB annotations and quick_series
func ProcessTag ¶ added in v0.5.0
ProcessTag - insert given Tag into Postgres TSDB
func ProgressInfo ¶
ProgressInfo display info about progress: i/n if current time >= last + period If displayed info, update last
func PullRequestIDOrNil ¶
func PullRequestIDOrNil(prPtr *PullRequest) interface{}
PullRequestIDOrNil - return PullRequest ID from pointer or nil
func QuarterStart ¶
QuarterStart - return time rounded to current month start
func QueryRowSQL ¶
QueryRowSQL executes given SQL on Postgres DB (and returns single row)
func QueryRowSQLTx ¶ added in v0.5.0
QueryRowSQLTx executes given SQL on Postgres DB (and returns single row)
func QuerySQL ¶
QuerySQL executes given SQL on Postgres DB (and returns rowset that needs to be closed)
func QuerySQLTx ¶
QuerySQLTx executes given SQL on Postgres DB (and returns rowset that needs to be closed) It is for running inside transaction
func QuerySQLTxWithErr ¶
QuerySQLTxWithErr wrapper to QuerySQLTx that exists on error It is for running inside transaction
func QuerySQLWithErr ¶
QuerySQLWithErr wrapper to QuerySQL that exists on error
func ReadFile ¶ added in v0.5.0
ReadFile tries to read any filename, but have a fallback it attempts to replace current project name with shared: /proj/ -> /shared/ This is to allow reading files that can be shared between projects
func ReleaseIDOrNil ¶
func ReleaseIDOrNil(relPtr *Release) interface{}
ReleaseIDOrNil - return Release ID from pointer or nil
func RepoIDOrNil ¶
func RepoIDOrNil(repoPtr *Repo) interface{}
RepoIDOrNil - return Repo ID from pointer or nil
func RepoNameOrNil ¶
func RepoNameOrNil(repoPtr *Repo) interface{}
RepoNameOrNil - return Repo Name from pointer or nil
func SkipEmpty ¶
SkipEmpty - skip one element arrays contining only empty string This is what strings.Split() returns for empty input We expect empty array or empty map returned in such cases
func Slugify ¶ added in v0.5.0
Slugify replace all whitespace with "-", remove all non-word letters downcase
func StringOrNil ¶
func StringOrNil(strPtr *string) interface{}
StringOrNil - return either nil or value of strPtr
func StringsMapToArray ¶
StringsMapToArray this is a function that calls given function for all array items and returns array of items processed by this func Example call: lib.StringsMapToArray(func(x string) string { return strings.TrimSpace(x) }, []string{" a", " b ", "c "})
func StringsMapToSet ¶
StringsMapToSet this is a function that calls given function for all array items and returns set of items processed by this func Example call: lib.StringsMapToSet(func(x string) string { return strings.TrimSpace(x) }, []string{" a", " b ", "c "})
func StringsSetKeys ¶
StringsSetKeys - returns all keys from string map
func StripUnicode ¶
StripUnicode strip non-unicode and control characters from a string From: https://rosettacode.org/wiki/Strip_control_codes_and_extended_characters_from_a_string#Go
func Structure ¶
func Structure(ctx *Ctx)
Structure creates full database structure, indexes, views/summary tables etc
func TableColumnExists ¶ added in v0.5.0
TableColumnExists - checks if a given table's has a given column
func TableExists ¶ added in v0.5.0
TableExists - checks if a given table exists
func TimeParseAny ¶
TimeParseAny - attempts to parse time from string YYYY-MM-DD HH:MI:SS Skipping parts from right until only YYYY id left
func ToYMDHDate ¶
ToYMDHDate - return time formatted as YYYY-MM-DD HH
func ToYMDHMSDate ¶
ToYMDHMSDate - return time formatted as YYYY-MM-DD HH:MI:SS
func TruncStringOrNil ¶
TruncStringOrNil - return either nil or value of strPtr truncated to maxLen chars
func TruncToBytes ¶
TruncToBytes - truncates text to <= size bytes (note that this can be a lot less UTF-8 runes)
func WeekStart ¶
WeekStart - return time rounded to current week start Assumes first week day is Sunday
func WriteTSPoints ¶ added in v0.5.0
WriteTSPoints write batch of points to postgresql use mergeSeries = "name" to put all series in "name" table, and create "series" column that conatins all point names.
without merge, alee names will create separate tables.
use non-null mut when you are using this function from multiple threads that write to the same series name at the same time
use non-null mut only then.
No more giant lock approach here, but it is up to user to spcify call context, especially 2 last parameters!
Types ¶
type Actor ¶
Actor - GHA Actor structure Name is unexported and not used by JSON load/save But is used when importing affiliations from cncf/gitdm:github_users.json
type AllProjects ¶
AllProjects contain all projects data
type Annotation ¶
Annotation contain each annotation data
type Annotations ¶
type Annotations struct {
Annotations []Annotation
}
Annotations contain list of annotations
func GetAnnotations ¶
func GetAnnotations(ctx *Ctx, orgRepo, annoRegexp string) (annotations Annotations)
GetAnnotations queries uses `git` to get `orgRepo` all tags list for all tags and returns those matching `annoRegexp`
func GetFakeAnnotations ¶
func GetFakeAnnotations(startDate, joinDate time.Time) (annotations Annotations)
GetFakeAnnotations - returns 'startDate - joinDate' and 'joinDate - now' annotations
type AnnotationsByDate ¶
type AnnotationsByDate []Annotation
AnnotationsByDate annotations Sort interface
func (AnnotationsByDate) Len ¶
func (a AnnotationsByDate) Len() int
func (AnnotationsByDate) Less ¶
func (a AnnotationsByDate) Less(i, j int) bool
func (AnnotationsByDate) Swap ¶
func (a AnnotationsByDate) Swap(i, j int)
type Asset ¶
type Asset struct { ID int `json:"id"` CreatedAt time.Time `json:"created_at"` UpdatedAt time.Time `json:"updated_at"` Name string `json:"name"` Label *string `json:"label"` Uploader Actor `json:"uploader"` ContentType string `json:"content_type"` State string `json:"state"` Size int `json:"size"` DownloadCount int `json:"download_count"` }
Asset - GHA Asset structure
type Branch ¶
type Branch struct { SHA string `json:"sha"` User *Actor `json:"user"` Repo *Forkee `json:"repo"` // This is confusing, but actually GHA has "repo" fields that holds "forkee" structure Label string `json:"label"` Ref string `json:"ref"` }
Branch - GHA Branch structure
type Comment ¶
type Comment struct { ID int `json:"id"` Body string `json:"body"` CreatedAt time.Time `json:"created_at"` UpdatedAt time.Time `json:"updated_at"` User Actor `json:"user"` CommitID *string `json:"commit_id"` OriginalCommitID *string `json:"original_commit_id"` DiffHunk *string `json:"diff_hunk"` Position *int `json:"position"` OriginalPosition *int `json:"original_position"` Path *string `json:"path"` PullRequestReviewID *int `json:"pull_request_review_id"` Line *int `json:"line"` }
Comment - GHA Comment structure
type Commit ¶
type Commit struct { SHA string `json:"sha"` Author Author `json:"author"` Message string `json:"message"` Distinct bool `json:"distinct"` }
Commit - GHA Commit structure
type Ctx ¶
type Ctx struct { Debug int // From GHA2DB_DEBUG Debug level: 0-no, 1-info, 2-verbose, including SQLs, default 0 CmdDebug int // From GHA2DB_CMDDEBUG Commands execution Debug level: 0-no, 1-only output commands, 2-output commands and their output, 3-output full environment as well, default 0 JSONOut bool // From GHA2DB_JSON gha2db: write JSON files? default false DBOut bool // From GHA2DB_NODB gha2db: write to SQL database, default true ST bool // From GHA2DB_ST true: use single threaded version, false: use multi threaded version, default false NCPUs int // From GHA2DB_NCPUS, set to override number of CPUs to run, this overwrites GHA2DB_ST, default 0 (which means do not use it) PgHost string // From PG_HOST, default "localhost" PgPort string // From PG_PORT, default "5432" PgDB string // From PG_DB, default "gha" PgUser string // From PG_USER, default "gha_admin" PgPass string // From PG_PASS, default "password" PgSSL string // From PG_SSL, default "disable" Index bool // From GHA2DB_INDEX Create DB index? default false Table bool // From GHA2DB_SKIPTABLE Create table structure? default true Tools bool // From GHA2DB_SKIPTOOLS Create DB tools (like views, summary tables, materialized views etc)? default true Mgetc string // From GHA2DB_MGETC Character returned by mgetc (if non empty), default "" QOut bool // From GHA2DB_QOUT output all SQL queries?, default false CtxOut bool // From GHA2DB_CTXOUT output all context data (this struct), default false LogTime bool // From GHA2DB_SKIPTIME, output time with all lib.Printf(...) calls, default true, use GHA2DB_SKIPTIME to disable DefaultStartDate time.Time // From GHA2DB_STARTDT, default `2014-01-01 00:00 UTC`, expects format "YYYY-MM-DD HH:MI:SS", can be set in `projects.yaml` via `start_date:`, value from projects.yaml (if set) has the highest priority. ForceStartDate bool // From GHA2DB_STARTDT_FORCE, default false LastSeries string // From GHA2DB_LASTSERIES, use this TSDB series to determine last timestamp date, default "events_h" SkipTSDB bool // From GHA2DB_SKIPTSDB gha2db_sync tool, skip TS DB processing? for calc_metric it skips final series write, default false SkipPDB bool // From GHA2DB_SKIPPDB gha2db_sync tool, skip Postgres DB processing? default false ResetTSDB bool // From GHA2DB_RESETTSDB sync tool, regenerate all TS points? default false ResetRanges bool // From GHA2DB_RESETRANGES sync tool, regenerate all past quick ranges? default false Explain bool // From GHA2DB_EXPLAIN runq tool, prefix query with "explain " - it will display query plan instead of executing real query, default false OldFormat bool // From GHA2DB_OLDFMT gha2db tool, if set then use pre 2015 GHA JSONs format Exact bool // From GHA2DB_EXACT gha2db tool, if set then orgs list provided from commandline is used as a list of exact repository full names, like "a/b,c/d,e", if not only full names "a/b,x/y" can be treated like this, names without "/" are either orgs or repos. LogToDB bool // From GHA2DB_SKIPLOG all tools, if set, DB logging into Postgres table `gha_logs` in `devstats` database will be disabled Local bool // From GHA2DB_LOCAL gha2db_sync tool, if set, gha2_db will call other tools prefixed with "./" to use local compile ones. Otherwise it will call binaries without prefix (so it will use thos ein /usr/bin/). MetricsYaml string // From GHA2DB_METRICS_YAML gha2db_sync tool, set other metrics.yaml file, default is "metrics/{{project}}metrics.yaml" TagsYaml string // From GHA2DB_TAGS_YAML tags tool, set other tags.yaml file, default is "metrics/{{project}}/tags.yaml" ColumnsYaml string // From GHA2DB_COLUMNS_YAML tags tool, set other columns.yaml file, default is "metrics/{{project}}/columns.yaml" VarsYaml string // From GHA2DB_VARS_YAML db_vars tool, set other vars.yaml file, default is "metrics/{{project}}/vars.yaml" GitHubOAuth string // From GHA2DB_GITHUB_OAUTH ghapi2db tool, if not set reads from /etc/github/oauth file, set to "-" to force public access. ClearDBPeriod string // From GHA2DB_MAXLOGAGE gha2db_sync tool, maximum age of devstats.gha_logs entries, default "1 week" Trials []int // From GHA2DB_TRIALS, all Postgres related tools, retry periods for "too many connections open" error WebHookRoot string // From GHA2DB_WHROOT, webhook tool, default "/hook", must match .travis.yml notifications webhooks WebHookPort string // From GHA2DB_WHPORT, webhook tool, default ":1982", note that webhook listens using http:1982, but we use apache on https:2982 (to enable https protocol and proxy requests to http:1982) WebHookHost string // From GHA2DB_WHHOST, webhook tool, default "127.0.0.1" (this can be localhost to disable access by IP, we use Apache proxy to enable https and then apache only need 127.0.0.1) CheckPayload bool // From GHA2DB_SKIP_VERIFY_PAYLOAD, webhook tool, default true, use GHA2DB_SKIP_VERIFY_PAYLOAD=1 to manually test payloads FullDeploy bool // From GHA2DB_SKIP_FULL_DEPLOY, webhook tool, default true, use GHA2DB_SKIP_FULL_DEPLOY=1 to ignore "[deploy]" requests that call `./devel/deploy_all.sh`. DeployBranches []string // From GHA2DB_DEPLOY_BRANCHES, webhook tool, default "master" - comma separated list DeployStatuses []string // From GHA2DB_DEPLOY_STATUSES, webhook tool, default "Passed,Fixed", - comma separated list DeployResults []int // From GHA2DB_DEPLOY_RESULTS, webhook tool, default "0", - comma separated list DeployTypes []string // From GHA2DB_DEPLOY_TYPES, webhook tool, default "push", - comma separated list ProjectRoot string // From GHA2DB_PROJECT_ROOT, webhook tool, no default, must be specified to run webhook tool ExecFatal bool // default true, set this manually to false to avoid lib.ExecCommand calling os.Exit() on failure and return error instead ExecQuiet bool // default false, set this manually to true to have quite exec failures (for example `get_repos` git-clones or git-pulls on errors). ExecOutput bool // default false, set to true to capture commands STDOUT Project string // From GHA2DB_PROJECT, gha2db_sync default "", You should set it to something like "kubernetes", "prometheus" etc. TestsYaml string // From GHA2DB_TESTS_YAML ./dbtest.sh tool, set other tests.yaml file, default is "tests.yaml" ReposDir string // From GHA2DB_REPOS_DIR get_repos tool, default "~/devstats_repos/" ProcessRepos bool // From GHA2DB_PROCESS_REPOS get_repos tool, enable processing (cloning/pulling) all devstats repos, default false ProcessCommits bool // From GHA2DB_PROCESS_COMMITS get_repos tool, enable update/create mapping table: commit - list of file that commit refers to, default false ExternalInfo bool // From GHA2DB_EXTERNAL_INFO get_repos tool, enable outputing data needed by external tools (cncf/gitdm), default false ProjectsCommits string // From GHA2DB_PROJECTS_COMMITS get_repos tool, set list of projects for commits analysis instead of analysing all, default "" - means all ProjectsYaml string // From GHA2DB_PROJECTS_YAML, many tools - set main projects file, default "projects.yaml" ProjectsOverride map[string]bool // From GHA2DB_PROJECTS_OVERRIDE, get_repos and ./devstats tools - for example "-pro1,+pro2" means never sync pro1 and always sync pro2 (even if disabled in `projects.yaml`). ExcludeRepos map[string]bool // From GHA2DB_EXCLUDE_REPOS, gha2db tool, default "" - comma separated list of repos to exclude, example: "theupdateframework/notary,theupdateframework/other" InputDBs []string // From GHA2DB_INPUT_DBS, merge_dbs tool - list of input databases to merge, order matters - first one will insert on a clean DB, next will do insert ignore (to avoid constraints failure due to common data) OutputDB string // From GHA2DB_OUTPUT_DB, merge_dbs tool - output database to merge into TmOffset int // From GHA2DB_TMOFFSET, gha2db_sync tool - uses time offset to decide when to calculate various metrics, default offset is 0 which means UTC, good offset for USA is -6, and for Poland is 1 or 2 DefaultHostname string // "devstats.cncf.io" RecentRange string // From GHA2DB_RECENT_RANGE, ghapi2db tool, default '2 hours'. This is a recent period to check open issues/PR to fix their labels and milestones. MinGHAPIPoints int // From GHA2DB_MIN_GHAPI_POINTS, ghapi2db tool, minimum GitHub API points, before waiting for reset. MaxGHAPIWaitSeconds int // From GHA2DB_MAX_GHAPI_WAIT, ghapi2db tool, maximum wait time for GitHub API points reset (in seconds). MaxGHAPIRetry int // From GHA2DB_MAX_GHAPI_RETRY, ghapi2db tool, maximum wait retries SkipGHAPI bool // From GHA2DB_GHAPISKIP, ghapi2db tool, if set then tool is not creating artificial events using GitHub API SkipArtificailClean bool // From GHA2DB_AECLEANSKIP, ghapi2db tool, if set then tool is not attempting to clean unneeded artificial events SkipGetRepos bool // From GHA2DB_GETREPOSSKIP, get_repos tool, if set then tool does nothing OnlyIssues []int64 // From GHA2DB_ONLY_ISSUES, ghapi2db tool, process a user provided list of issues "issue_id1,issue_id2,...,issue_idN", default "". This is for GH API debugging. OnlyEvents []int64 // From GHA2DB_ONLY_EVENTS, ghapi2db tool, process a user provided list of events "event_id1,event_id2,...,event_idN", default "". This is for artificial events cleanup debugging. CSVFile string // From GHA2DB_CSVOUT, runq tool, if set, saves result in this file ComputeAll bool // From GHA2DB_COMPUTE_ALL, all tools, if set then no period decisions are taken based on time, but all possible periods are recalculated ActorsFilter bool // From GHA2DB_ACTORS_FILTER gha2db tool, if enabled then actor filterning will be added, default false ActorsAllow *regexp.Regexp // From GHA2DB_ACTORS_ALLOW, gha2db tool, process JSON if actor matches this regexp, default "" ActorsForbid *regexp.Regexp // From GHA2DB_ACTORS_FORBID, gha2db tool, process JSON if actor matches this regexp, default "" OnlyMetrics map[string]bool // From GHA2DB_ONLY_METRICS, gha2db_sync tool, default "" - comma separated list of metrics to process, as fiven my "sql: name" in the "metrics.yaml" file. Only those metrics will be calculated. }
Ctx - environment context packed in structure
type Dummy ¶
type Dummy struct{}
Dummy - structure with no data pointer to this struct is used to test if such field was present in JSON or not
type Event ¶
type Event struct { ID string `json:"id"` Type string `json:"type"` Public bool `json:"public"` CreatedAt time.Time `json:"created_at"` Actor Actor `json:"actor"` Repo Repo `json:"repo"` Org *Org `json:"org"` Payload Payload `json:"payload"` }
Event - full GHA (GitHub Archive) event structure
type EventOld ¶
type EventOld struct { ID string `json:"-"` Type string `json:"type"` Public bool `json:"public"` CreatedAt time.Time `json:"created_at"` Actor string `json:"actor"` Repository ForkeeOld `json:"repository"` Payload *PayloadOld `json:"payload"` }
EventOld - full GHA (GitHub Archive) event structure, before 2015
type Forkee ¶
type Forkee struct { ID int `json:"id"` Name string `json:"name"` FullName string `json:"full_name"` Owner Actor `json:"owner"` Description *string `json:"description"` Public *bool `json:"public"` Fork bool `json:"fork"` CreatedAt time.Time `json:"created_at"` UpdatedAt time.Time `json:"updated_at"` PushedAt *time.Time `json:"pushed_at"` Homepage *string `json:"homepage"` Size int `json:"size"` StargazersCount int `json:"stargazers_count"` HasIssues bool `json:"has_issues"` HasProjects *bool `json:"has_projects"` HasDownloads bool `json:"has_downloads"` HasWiki bool `json:"has_wiki"` HasPages *bool `json:"has_pages"` Forks int `json:"forks"` OpenIssues int `json:"open_issues"` Watchers int `json:"watchers"` DefaultBranch string `json:"default_branch"` }
Forkee - GHA Forkee structure
type ForkeeOld ¶
type ForkeeOld struct { ID int `json:"id"` CreatedAt time.Time `json:"created_at"` Description *string `json:"description"` Fork bool `json:"fork"` Forks int `json:"forks"` HasDownloads bool `json:"has_downloads"` HasIssues bool `json:"has_issues"` HasWiki bool `json:"has_wiki"` Homepage *string `json:"homepage"` Language *string `json:"language"` DefaultBranch string `json:"master_branch"` Name string `json:"name"` OpenIssues int `json:"open_issues"` Organization *string `json:"organization"` Owner string `json:"owner"` Private *bool `json:"private"` PushedAt *time.Time `json:"pushed_at"` Size int `json:"size"` Stargazers int `json:"stargazers"` Watchers int `json:"watchers"` }
ForkeeOld - GHA Forkee structure (from before 2015) Handle missing 4 last properties (including two non-nulls!)
type Issue ¶
type Issue struct { ID int `json:"id"` Number int `json:"number"` Comments int `json:"comments"` Title string `json:"title"` State string `json:"state"` Locked bool `json:"locked"` Body *string `json:"body"` User Actor `json:"user"` Assignee *Actor `json:"assignee"` Labels []Label `json:"labels"` Assignees []Actor `json:"assignees"` Milestone *Milestone `json:"milestone"` CreatedAt time.Time `json:"created_at"` UpdatedAt time.Time `json:"updated_at"` ClosedAt *time.Time `json:"closed_at"` PullRequest *Dummy `json:"pull_request"` }
Issue - GHA Issue structure
type Label ¶
type Label struct { ID *int `json:"id"` Name string `json:"name"` Color string `json:"color"` Default *bool `json:"default"` }
Label - GHA Label structure
type Milestone ¶
type Milestone struct { ID int `json:"id"` Name string `json:"name"` Number int `json:"number"` Title string `json:"title"` Description *string `json:"description"` Creator *Actor `json:"creator"` OpenIssues int `json:"open_issues"` ClosedIssues int `json:"closed_issues"` State string `json:"state"` CreatedAt time.Time `json:"created_at"` UpdatedAt time.Time `json:"updated_at"` ClosedAt *time.Time `json:"closed_at"` DueOn *time.Time `json:"due_on"` }
Milestone - GHA Milestone structure
type Page ¶
type Page struct { SHA string `json:"sha"` Action string `json:"action"` Title string `json:"title"` }
Page - GHA Page structure
type Payload ¶
type Payload struct { PushID *int `json:"push_id"` Size *int `json:"size"` Ref *string `json:"ref"` Head *string `json:"head"` Before *string `json:"before"` Action *string `json:"action"` RefType *string `json:"ref_type"` MasterBranch *string `json:"master_branch"` Description *string `json:"description"` Number *int `json:"number"` Forkee *Forkee `json:"forkee"` Release *Release `json:"release"` Member *Actor `json:"member"` Issue *Issue `json:"issue"` Comment *Comment `json:"comment"` Commits *[]Commit `json:"commits"` Pages *[]Page `json:"pages"` PullRequest *PullRequest `json:"pull_request"` }
Payload - GHA Payload structure
type PayloadOld ¶
type PayloadOld struct { Issue *int `json:"issue"` IssueID *int `json:"issue_id"` Comment *Comment `json:"comment"` CommentID *int `json:"comment_id"` Description *string `json:"description"` MasterBranch *string `json:"master_branch"` Ref *string `json:"ref"` Action *string `json:"action"` RefType *string `json:"ref_type"` Head *string `json:"head"` Size *int `json:"size"` Number *int `json:"number"` PullRequest *PullRequest `json:"pull_request"` Member *Actor `json:"member"` Release *Release `json:"release"` Pages *[]Page `json:"pages"` Commit *string `json:"commit"` SHAs *[]interface{} `json:"shas"` Repository *Forkee `json:"repository"` Team *Team `json:"team"` }
PayloadOld - GHA Payload structure (from before 2015)
type Project ¶
type Project struct { CommandLine []string `yaml:"command_line"` StartDate *time.Time `yaml:"start_date"` PDB string `yaml:"psql_db"` Disabled bool `yaml:"disabled"` MainRepo string `yaml:"main_repo"` AnnotationRegexp string `yaml:"annotation_regexp"` Order int `yaml:"order"` JoinDate *time.Time `yaml:"join_date"` FilesSkipPattern string `yaml:"files_skip_pattern"` Env map[string]string `yaml:"env"` }
Project contain mapping from project name to its command line used to sync it
type PullRequest ¶
type PullRequest struct { ID int `json:"id"` Base Branch `json:"base"` Head Branch `json:"head"` User Actor `json:"user"` Number int `json:"number"` State string `json:"state"` Locked *bool `json:"locked"` Title string `json:"title"` Body *string `json:"body"` CreatedAt time.Time `json:"created_at"` UpdatedAt time.Time `json:"updated_at"` ClosedAt *time.Time `json:"closed_at"` MergedAt *time.Time `json:"merged_at"` MergeCommitSHA *string `json:"merge_commit_sha"` Assignee *Actor `json:"assignee"` Assignees *[]Actor `json:"assignees"` RequestedReviewers *[]Actor `json:"requested_reviewers"` Milestone *Milestone `json:"milestone"` Merged *bool `json:"merged"` Mergeable *bool `json:"mergeable"` MergedBy *Actor `json:"merged_by"` MergeableState *string `json:"mergeable_state"` Rebaseable *bool `json:"rebaseable"` Comments *int `json:"comments"` ReviewComments *int `json:"review_comments"` MaintainerCanModify *bool `json:"maintainer_can_modify"` Commits *int `json:"commits"` Additions *int `json:"additions"` Deletions *int `json:"deletions"` ChangedFiles *int `json:"changed_files"` }
PullRequest - GHA Pull Request structure
type Release ¶
type Release struct { ID int `json:"id"` TagName string `json:"tag_name"` TargetCommitish string `json:"target_commitish"` Name *string `json:"name"` Draft bool `json:"draft"` Author Actor `json:"author"` Prerelease bool `json:"prerelease"` CreatedAt time.Time `json:"created_at"` PublishedAt *time.Time `json:"published_at"` Body *string `json:"body"` Assets []Asset `json:"assets"` }
Release - GHA Release structure
type TSPoint ¶ added in v0.5.0
type TSPoint struct {
// contains filtered or unexported fields
}
TSPoint keeps single time series point
type TSPoints ¶ added in v0.5.0
type TSPoints []TSPoint
TSPoints keeps batch of TSPoint values to write
type Tag ¶ added in v0.5.0
type Tag struct { Name string `yaml:"name"` SQLFile string `yaml:"sql"` SeriesName string `yaml:"series_name"` NameTag string `yaml:"name_tag"` ValueTag string `yaml:"value_tag"` OtherTags map[string]string `yaml:"other_tags"` }
Tag contain each TSDB tag data