dads

package module

v0.0.0-...-b175f30 Latest Latest Go to latest Published: Jun 23, 2022 License: Apache-2.0 Imports: 43 Imported by: 2

Details

Valid go.mod file
Redistributable license
Tagged version
Stable version
Learn more about best practices

Repository

github.com/LF-Engineering/da-ds

Links

Open Source Insights

README ¶

dads

DevAnalytics data source

Jira: implemented
Groups.io: wip

Documentation ¶

Index ¶

Constants
Variables
func AddRedacted(newRedacted string, useMutex bool)
func AffsDataForRoles(ctx *Ctx, ds DS, rich map[string]interface{}, roles []string) (data map[string]interface{}, e error)
func AffsIdentityIDs(ctx *Ctx, ds DS, identity map[string]interface{}) (ids [2]interface{})
func AnonymizeURL(url string) string
func Base64DecodeCookies(enc []byte) (cookies []string, err error)
func Base64DecodeHeaders(enc []byte) (headers map[string][]string, err error)
func Base64EncodeCookies(cookies []string) (enc []byte)
func Base64EncodeHeaders(headers map[string][]string) (enc []byte)
func BytesToStringTrunc(data []byte, maxLen int, addLenInfo bool) (str string)
func CacheSummary(ctx *Ctx)
func CommonFields(ds DS, date interface{}, category string) (fields map[string]interface{})
func ConfluenceEnrichItemsFunc(ctx *Ctx, ds DS, thrN int, items []interface{}, docs *[]interface{}) (err error)
func ConnectAffiliationsDB(ctx *Ctx)
func CookieToString(c *http.Cookie) (s string)
func CopyAffsRoleData(dst, src map[string]interface{}, dstRole, srcRole string)
func CreateESCache(ctx *Ctx)
func DBUploadIdentitiesFunc(ctx *Ctx, ds DS, thrN int, docs, outDocs *[]interface{}, last bool) (e error)
func DeepSet(m interface{}, ks []string, v interface{}, create bool) (err error)
func Dig(iface interface{}, keys []string, fatal, silent bool) (v interface{}, ok bool)
func DumpKeys(i interface{}) string
func DumpPreview(i interface{}, l int) string
func ESBulkUploadFunc(ctx *Ctx, ds DS, thrN int, docs, outDocs *[]interface{}, last bool) (e error)
func ESCacheDelete(ctx *Ctx, key string)
func ESCacheDeleteExpired(ctx *Ctx)
func ESCacheSet(ctx *Ctx, key string, entry *ESCacheEntry)
func EmptyAffsItem(role string, undef bool) map[string]interface{}
func Enrich(ctx *Ctx, ds DS) (err error)
func EnrichComments(ctx *Ctx, ds DS, comments []interface{}, item map[string]interface{}, ...) (richComments []interface{}, err error)
func EnrichItem(ctx *Ctx, ds DS, richItem map[string]interface{}) (err error)
func EnrichPairProgrammingItem(richItem map[string]interface{}) (err error)
func EnsurePath(path string, noLastDir bool) (string, error)
func ExecCommand(ctx *Ctx, cmdAndArgs []string, cwd string, env map[string]string) (sout, serr string, err error)
func ExecCommandPipe(ctx *Ctx, cmdAndArgs []string, cwd string, env map[string]string) (stdOutPipe io.ReadCloser, cmd *exec.Cmd, err error)
func ExecDB(ctx *Ctx, query string, args ...interface{}) (res sql.Result, err error)
func ExecSQL(ctx *Ctx, tx *sql.Tx, query string, args ...interface{}) (sql.Result, error)
func ExecTX(ctx *Ctx, tx *sql.Tx, query string, args ...interface{}) (res sql.Result, err error)
func ExecuteAffiliationsAPICall(ctx *Ctx, method, path string, cacheToken bool) (data map[string]interface{}, err error)
func FatalOnError(err error) string
func Fatalf(f string, a ...interface{})
func FetchRaw(ctx *Ctx, ds DS) (err error)
func FilterRedacted(str string) string
func FindObject(ctx *Ctx, object, key, id string, fields []string) (obj map[string]interface{}, err error)
func ForEachESItem(ctx *Ctx, ds DS, raw bool, ...) (err error)
func GerritEnrichItemsFunc(ctx *Ctx, ds DS, thrN int, items []interface{}, docs *[]interface{}) (err error)
func GetAPIToken() (string, error)
func GetESCache(ctx *Ctx, k string) (b []byte, tg string, expires time.Time, ok bool)
func GetEnrollments(ctx *Ctx, ds DS, uuid string, dt time.Time, single bool) (orgs []string, e error)
func GetEnrollmentsBoth(ctx *Ctx, ds DS, uuid string, dt time.Time) (org string, orgs []string, e error)
func GetEnrollmentsMulti(ctx *Ctx, ds DS, uuid string, dt time.Time) (orgs []string, e error)
func GetEnrollmentsSingle(ctx *Ctx, ds DS, uuid string, dt time.Time) (org string, e error)
func GetIdentityUUID(ctx *Ctx, ds DS, id string) (uuid interface{})
func GetL2Cache(ctx *Ctx, k string) (b []byte, ok bool)
func GetLastOffset(ctx *Ctx, ds DS, raw bool) (offset float64)
func GetLastUpdate(ctx *Ctx, ds DS, raw bool) (lastUpdate *time.Time)
func GetRedacted() (str string)
func GetThreadsNum(ctx *Ctx) int
func GitEnrichItemsFunc(ctx *Ctx, ds DS, thrN int, items []interface{}, docs *[]interface{}) (err error)
func GitHubEnrichItemsFunc(ctx *Ctx, ds DS, thrN int, items []interface{}, docs *[]interface{}) (err error)
func GroupsioEnrichItemsFunc(ctx *Ctx, ds DS, thrN int, items []interface{}, docs *[]interface{}) (err error)
func HandleMapping(ctx *Ctx, ds DS, raw bool) (err error)
func IdentityAffsData(ctx *Ctx, ds DS, identity map[string]interface{}, aid interface{}, ...) (outItem map[string]interface{}, empty bool, e error)
func IdentityAffsDomain(identity map[string]interface{}) (domain interface{}, email string)
func IndexAt(s, sep string, n int) int
func InitializeAuth0() error
func InterfaceToStringTrunc(iface interface{}, maxLen int, addLenInfo bool) (str string)
func IsOldFormat(ctx *Ctx, idx string) (old bool)
func IsValidDomain(domain string) (valid bool)
func IsValidEmail(email string, validateDomain, guess bool) (valid bool, newEmail string)
func ItemsIdentitiesFunc(ctx *Ctx, ds DS, thrN int, items []interface{}, docs *[]interface{}) (err error)
func ItemsRefreshIdentitiesFunc(ctx *Ctx, ds DS, thrN int, richItems []interface{}, docs *[]interface{}) (err error)
func JSONEscape(str string) string
func JiraEnrichItemsFunc(ctx *Ctx, ds DS, thrN int, items []interface{}, docs *[]interface{}) (err error)
func KeysOnly(i interface{}) (o map[string]interface{})
func MatchGroups(re *regexp.Regexp, arg string) (result map[string]string)
func MatchGroupsArray(re *regexp.Regexp, arg string) (result map[string][]string)
func MaybeESCacheCleanup(ctx *Ctx)
func MaybeMemCacheCleanup(ctx *Ctx)
func MemCacheDeleteExpired(ctx *Ctx)
func NoSSLVerify()
func ParseAddresses(ctx *Ctx, addrs string, maxAddrs int) (emails []*mail.Address, ok bool)
func ParseDateWithTz(indt string) (dt, dtInTz time.Time, off float64, valid bool)
func ParseMBoxMsg(ctx *Ctx, groupName string, msg []byte) (item map[string]interface{}, valid, warn bool)
func PartitionString(s string, sep string) [3]string
func PeriodParse(perStr string) (dur time.Duration, ok bool)
func PostprocessFields(outItem map[string]interface{}, role, email string)
func PostprocessNameUsername(name, username, email string) (outName, outUsername string)
func PrettyPrintJSON(jsonBytes []byte) []byte
func PreviewOnly(i interface{}, l int) (o interface{})
func PrintCache(iCache interface{}) (s string)
func Printf(format string, args ...interface{})
func PrintfNoRedacted(format string, args ...interface{})
func ProgressInfo(i, n int, start time.Time, last *time.Time, period time.Duration, msg string)
func QueryDB(ctx *Ctx, query string, args ...interface{}) (rows *sql.Rows, err error)
func QueryOut(ctx *Ctx, in bool, err error, query string, args ...interface{})
func QuerySQL(ctx *Ctx, tx *sql.Tx, query string, args ...interface{}) (*sql.Rows, error)
func QueryTX(ctx *Ctx, tx *sql.Tx, query string, args ...interface{}) (rows *sql.Rows, err error)
func QueryToStringArray(ctx *Ctx, query string, args ...interface{}) (res []string)
func QueryToStringIntArrays(ctx *Ctx, query string, args ...interface{}) (sa []string, ia []int64)
func RedactEmail(in, suff string, forceSuff bool) string
func RefreshIdentities(ctx *Ctx, ds DS) (err error)
func Request(ctx *Ctx, url, method string, headers map[string]string, payload []byte, ...) (result interface{}, status int, outCookies []string, ...)
func RequestNoRetry(ctx *Ctx, url, method string, headers map[string]string, payload []byte, ...) (result interface{}, status int, isJSON bool, outCookies []string, ...)
func ResetThreadsNum(ctx *Ctx)
func ResetUUIDCache()
func RocketchatEnrichItemsFunc(ctx *Ctx, ds DS, thrN int, items []interface{}, docs *[]interface{}) (err error)
func SendMultipleDocumentsToGAP(ctx *Ctx, indexNames, docIDs []string, docs [][]byte, cacheToken bool) (err error)
func SendSingleDocumentToGAP(ctx *Ctx, indexName, docID string, doc []byte, cacheToken bool) (err error)
func SendToElastic(ctx *Ctx, ds DS, raw bool, key string, items []interface{}) (err error)
func SetDBSessionOrigin(ctx *Ctx) (err error)
func SetESCache(ctx *Ctx, k, tg string, b []byte, expires time.Duration)
func SetL2Cache(ctx *Ctx, k, tg string, b []byte, expires time.Duration)
func SetMT()
func SleepForRateLimit(ctx *Ctx, ds DS, rateLimit, rateLimitReset, minRate int, waitRate bool) (err error)
func StandardItemsFunc(ctx *Ctx, ds DS, items []interface{}, docs *[]interface{}) (err error)
func StringToBool(v string) bool
func StringToCookie(s string) (c *http.Cookie)
func StringTrunc(data string, maxLen int, addLenInfo bool) (str string)
func StubEnrichItemsFunc(ctx *Ctx, ds DS, thrN int, items []interface{}, docs *[]interface{}) (err error)
func TimeParseAny(dtStr string) (time.Time, error)
func TimeParseES(dtStr string) (time.Time, error)
func TimeParseESSec(dtStr string) (time.Time, error)
func TimeParseInterfaceString(date interface{}) (dt time.Time, err error)
func ToESDate(dt time.Time) string
func ToYMDDate(dt time.Time) string
func ToYMDHMDate(dt time.Time) string
func ToYMDHMSDate(dt time.Time) string
func ToYMDTHMSZDate(dt time.Time) string
func UUIDAffs(ctx *Ctx, args ...string) (h string)
func UUIDNonEmpty(ctx *Ctx, args ...string) (h string)
func UpdateRateLimit(ctx *Ctx, ds DS, headers map[string][]string, ...) (rateLimit, rateLimitReset, secondsToReset int)
func UploadIdentities(ctx *Ctx, ds DS) (err error)
type BugZilla
type Ctx
- func (ctx *Ctx) AffsDBConfigured() bool
- func (ctx *Ctx) BoolEnv(k string) bool
- func (ctx *Ctx) Env(v string) string
- func (ctx Ctx) Info() string
- func (ctx *Ctx) Init()
- func (ctx *Ctx) ParseFlags()
- func (ctx *Ctx) Print()
- func (ctx *Ctx) Validate() (err error)
type DS
type DSConfluence
- func (j *DSConfluence) AddMetadata(ctx *Ctx, item interface{}) (mItem map[string]interface{})
- func (j *DSConfluence) AffsItems(ctx *Ctx, page map[string]interface{}, roles []string, date interface{}) (affsItems map[string]interface{}, err error)
- func (j *DSConfluence) AllRoles(ctx *Ctx, item map[string]interface{}) ([]string, bool)
- func (j *DSConfluence) CalculateTimeToReset(ctx *Ctx, rateLimit, rateLimitReset int) (seconds int)
- func (j *DSConfluence) Categories() map[string]struct{}
- func (j *DSConfluence) CustomEnrich() bool
- func (j *DSConfluence) CustomFetchRaw() bool
- func (j *DSConfluence) DateField(*Ctx) string
- func (j *DSConfluence) ElasticRawMapping() []byte
- func (j *DSConfluence) ElasticRichMapping() []byte
- func (j *DSConfluence) Enrich(ctx *Ctx) (err error)
- func (j *DSConfluence) EnrichItem(ctx *Ctx, item map[string]interface{}, author string, affs bool, ...) (rich map[string]interface{}, err error)
- func (j *DSConfluence) EnrichItems(ctx *Ctx) (err error)
- func (j *DSConfluence) FetchItems(ctx *Ctx) (err error)
- func (j *DSConfluence) FetchRaw(ctx *Ctx) (err error)
- func (j *DSConfluence) GetConfluenceContents(ctx *Ctx, fromDate, next string) (contents []map[string]interface{}, newNext string, err error)
- func (j *DSConfluence) GetHistoricalContents(ctx *Ctx, content map[string]interface{}, dateFrom time.Time) (contents []map[string]interface{}, err error)
- func (j *DSConfluence) GetItemIdentities(ctx *Ctx, doc interface{}) (identities map[[3]string]struct{}, err error)
- func (j *DSConfluence) GetRoleIdentity(ctx *Ctx, item map[string]interface{}, role string) (identity map[string]interface{})
- func (j *DSConfluence) HasIdentities() bool
- func (j DSConfluence) Info() string
- func (j *DSConfluence) ItemCategory(item interface{}) string
- func (j *DSConfluence) ItemID(item interface{}) string
- func (j *DSConfluence) ItemUpdatedOn(item interface{}) time.Time
- func (j *DSConfluence) Name() string
- func (j *DSConfluence) OffsetField(*Ctx) string
- func (j *DSConfluence) Origin(ctx *Ctx) string
- func (j *DSConfluence) OriginField(ctx *Ctx) string
- func (j *DSConfluence) ParseArgs(ctx *Ctx) (err error)
- func (j *DSConfluence) ResumeNeedsCategory(ctx *Ctx, raw bool) bool
- func (j *DSConfluence) ResumeNeedsOrigin(ctx *Ctx, raw bool) bool
- func (j *DSConfluence) RichAuthorField(*Ctx) string
- func (j *DSConfluence) RichIDField(*Ctx) string
- func (j *DSConfluence) SupportDateFrom() bool
- func (j *DSConfluence) SupportOffsetFrom() bool
- func (j *DSConfluence) UseDefaultMapping(ctx *Ctx, raw bool) bool
- func (j *DSConfluence) Validate(ctx *Ctx) (err error)
type DSGerrit
- func (j *DSGerrit) AddMetadata(ctx *Ctx, item interface{}) (mItem map[string]interface{})
- func (j *DSGerrit) AffsItems(ctx *Ctx, review map[string]interface{}, roles []string, date interface{}) (affsItems map[string]interface{}, err error)
- func (j *DSGerrit) AllRoles(ctx *Ctx, rich map[string]interface{}) (roles []string, static bool)
- func (j *DSGerrit) CalculateTimeToReset(ctx *Ctx, rateLimit, rateLimitReset int) (seconds int)
- func (j *DSGerrit) Categories() map[string]struct{}
- func (j *DSGerrit) ConvertDates(ctx *Ctx, review map[string]interface{})
- func (j *DSGerrit) CustomEnrich() bool
- func (j *DSGerrit) CustomFetchRaw() bool
- func (j *DSGerrit) DateField(*Ctx) string
- func (j *DSGerrit) ElasticRawMapping() []byte
- func (j *DSGerrit) ElasticRichMapping() []byte
- func (j *DSGerrit) Enrich(ctx *Ctx) (err error)
- func (j *DSGerrit) EnrichApprovals(ctx *Ctx, review, patchSet map[string]interface{}, ...) (richItems []interface{}, err error)
- func (j *DSGerrit) EnrichComments(ctx *Ctx, review map[string]interface{}, comments []map[string]interface{}, ...) (richItems []interface{}, err error)
- func (j *DSGerrit) EnrichItem(ctx *Ctx, item map[string]interface{}, author string, affs bool, ...) (rich map[string]interface{}, err error)
- func (j *DSGerrit) EnrichItems(ctx *Ctx) (err error)
- func (j *DSGerrit) EnrichPatchsets(ctx *Ctx, review map[string]interface{}, patchSets []map[string]interface{}, ...) (richItems []interface{}, err error)
- func (j *DSGerrit) FetchItems(ctx *Ctx) (err error)
- func (j *DSGerrit) FetchRaw(ctx *Ctx) (err error)
- func (j *DSGerrit) FirstApprovalDatetime(ctx *Ctx, review map[string]interface{}, patchSets []interface{}) (approvalDatetime interface{})
- func (j *DSGerrit) FirstPatchsetApprovalDatetime(ctx *Ctx, patchSet map[string]interface{}) (approvalDatetime interface{})
- func (j *DSGerrit) FirstPatchsetReviewDatetime(ctx *Ctx, patchSet map[string]interface{}) (reviewDatetime interface{})
- func (j *DSGerrit) FirstReviewDatetime(ctx *Ctx, review map[string]interface{}, patchSets []interface{}) (reviewDatetime interface{})
- func (j *DSGerrit) GetGerritReviews(ctx *Ctx, after string, afterEpoch float64, startFrom int) (reviews []map[string]interface{}, newStartFrom int, err error)
- func (j *DSGerrit) GetGerritVersion(ctx *Ctx) (err error)
- func (j *DSGerrit) GetItemIdentities(ctx *Ctx, doc interface{}) (identities map[[3]string]struct{}, err error)
- func (j *DSGerrit) GetRoleIdentity(ctx *Ctx, item map[string]interface{}, role string) (identity map[string]interface{})
- func (j *DSGerrit) HasIdentities() bool
- func (j *DSGerrit) IdentityForObject(ctx *Ctx, obj map[string]interface{}) (identity [3]string)
- func (j DSGerrit) Info() string
- func (j *DSGerrit) InitGerrit(ctx *Ctx) (err error)
- func (j *DSGerrit) ItemCategory(item interface{}) string
- func (j *DSGerrit) ItemID(item interface{}) string
- func (j *DSGerrit) ItemUpdatedOn(item interface{}) time.Time
- func (j *DSGerrit) LastChangesetApprovalValue(ctx *Ctx, patchSets []interface{}) (status interface{})
- func (j *DSGerrit) Name() string
- func (j *DSGerrit) OffsetField(*Ctx) string
- func (j *DSGerrit) Origin(ctx *Ctx) string
- func (j *DSGerrit) OriginField(ctx *Ctx) string
- func (j *DSGerrit) ParseArgs(ctx *Ctx) (err error)
- func (j *DSGerrit) ResumeNeedsCategory(ctx *Ctx, raw bool) bool
- func (j *DSGerrit) ResumeNeedsOrigin(ctx *Ctx, raw bool) bool
- func (j *DSGerrit) RichAuthorField(*Ctx) string
- func (j *DSGerrit) RichIDField(*Ctx) string
- func (j *DSGerrit) SupportDateFrom() bool
- func (j *DSGerrit) SupportOffsetFrom() bool
- func (j *DSGerrit) UseDefaultMapping(ctx *Ctx, raw bool) bool
- func (j *DSGerrit) Validate(ctx *Ctx) (err error)
type DSGit
- func (j *DSGit) AddMetadata(ctx *Ctx, item interface{}) (mItem map[string]interface{})
- func (j *DSGit) AffsItems(ctx *Ctx, commit map[string]interface{}, roles []string, date interface{}) (affsItems map[string]interface{}, err error)
- func (j *DSGit) AllRoles(ctx *Ctx, item map[string]interface{}) (roles []string, static bool)
- func (j *DSGit) BuildCommit(ctx *Ctx) (commit map[string]interface{})
- func (j *DSGit) CalculateTimeToReset(ctx *Ctx, rateLimit, rateLimitReset int) (seconds int)
- func (j *DSGit) Categories() map[string]struct{}
- func (j *DSGit) CreateGitRepo(ctx *Ctx) (err error)
- func (j *DSGit) CustomEnrich() bool
- func (j *DSGit) CustomFetchRaw() bool
- func (j *DSGit) DateField(*Ctx) string
- func (j *DSGit) ElasticRawMapping() []byte
- func (j *DSGit) ElasticRichMapping() []byte
- func (j *DSGit) Enrich(ctx *Ctx) (err error)
- func (j *DSGit) EnrichItem(ctx *Ctx, item map[string]interface{}, skip string, affs bool, ...) (rich map[string]interface{}, err error)
- func (j *DSGit) EnrichItems(ctx *Ctx) (err error)
- func (*DSGit) ExtractPrevFileName(f string) (res string)
- func (j *DSGit) FetchItems(ctx *Ctx) (err error)
- func (j *DSGit) FetchRaw(ctx *Ctx) (err error)
- func (j *DSGit) GetAuthors(ctx *Ctx, m map[string]string, n map[string][]string) (authors map[string]struct{}, author string)
- func (j *DSGit) GetAuthorsData(ctx *Ctx, doc interface{}, auth string) (authorsMap map[string]struct{}, firstAuthor string)
- func (j *DSGit) GetCommitURL(origin, hash string) string
- func (j *DSGit) GetGitOps(ctx *Ctx, thrN int) (ch chan error, err error)
- func (j *DSGit) GetItemIdentities(ctx *Ctx, doc interface{}) (identities map[[3]string]struct{}, err error)
- func (j *DSGit) GetOrphanedCommits(ctx *Ctx, thrN int) (ch chan error, err error)
- func (j *DSGit) GetOtherPPAuthors(ctx *Ctx, doc interface{}) (othersMap map[string]map[string]struct{})
- func (j *DSGit) GetOtherTrailersAuthors(ctx *Ctx, doc interface{}) (othersMap map[string]map[[2]string]struct{})
- func (j *DSGit) GetRepoShortURL(origin string) (repoShortName string)
- func (j *DSGit) GetRoleIdentity(ctx *Ctx, commit map[string]interface{}, role string) map[string]interface{}
- func (j *DSGit) HandleRecentLines(line string)
- func (j *DSGit) HasIdentities() bool
- func (j *DSGit) IdentitiesFromGitAuthors(ctx *Ctx, authors map[string]struct{}) (identities map[[3]string]struct{})
- func (j *DSGit) IdentityFromGitAuthor(ctx *Ctx, author string) (identity [3]string)
- func (j DSGit) Info() string
- func (j *DSGit) ItemCategory(item interface{}) string
- func (j *DSGit) ItemID(item interface{}) string
- func (j *DSGit) ItemUpdatedOn(item interface{}) time.Time
- func (j *DSGit) MarkOrphanedCommits(ctx *Ctx) (err error)
- func (j *DSGit) Name() string
- func (j *DSGit) OffsetField(*Ctx) string
- func (j *DSGit) Origin(ctx *Ctx) string
- func (j *DSGit) OriginField(ctx *Ctx) string
- func (j *DSGit) PairProgrammingMetrics(ctx *Ctx, rich, commit map[string]interface{}) (err error)
- func (j *DSGit) ParseAction(ctx *Ctx, data map[string]string)
- func (j *DSGit) ParseArgs(ctx *Ctx) (err error)
- func (j *DSGit) ParseCommit(ctx *Ctx, line string) (parsed bool, err error)
- func (j *DSGit) ParseFile(ctx *Ctx, line string) (parsed, empty bool, err error)
- func (j *DSGit) ParseGitLog(ctx *Ctx) (cmd *exec.Cmd, err error)
- func (j *DSGit) ParseHeader(ctx *Ctx, line string) (parsed bool, err error)
- func (j *DSGit) ParseInit(ctx *Ctx, line string) (parsed bool, err error)
- func (j *DSGit) ParseMessage(ctx *Ctx, line string) (parsed bool, err error)
- func (j *DSGit) ParseNextCommit(ctx *Ctx) (commit map[string]interface{}, ok bool, err error)
- func (j *DSGit) ParseStats(ctx *Ctx, data map[string]string)
- func (j *DSGit) ParseTrailer(ctx *Ctx, line string)
- func (j *DSGit) ResumeNeedsCategory(ctx *Ctx, raw bool) bool
- func (j *DSGit) ResumeNeedsOrigin(ctx *Ctx, raw bool) bool
- func (j *DSGit) RichAuthorField(*Ctx) string
- func (j *DSGit) RichIDField(*Ctx) string
- func (j *DSGit) SetTLOC(ctx *Ctx) (err error)
- func (j *DSGit) SupportDateFrom() bool
- func (j *DSGit) SupportOffsetFrom() bool
- func (j *DSGit) TrailerDoc(ctx *Ctx, rich, item map[string]interface{}, author string) (trailer map[string]interface{}, skip bool, err error)
- func (j *DSGit) TrailerDocs(ctx *Ctx, rich map[string]interface{}) (trailers []map[string]interface{}, err error)
- func (j *DSGit) UniqueStringArray(ary []interface{}) []interface{}
- func (j *DSGit) UpdateGitRepo(ctx *Ctx) (err error)
- func (j *DSGit) UseDefaultMapping(ctx *Ctx, raw bool) bool
- func (j *DSGit) Validate(ctx *Ctx) (err error)
type DSGitHub
- func (j *DSGitHub) AddMetadata(ctx *Ctx, item interface{}) (mItem map[string]interface{})
- func (j *DSGitHub) AffsItems(ctx *Ctx, item map[string]interface{}, roles []string, date interface{}) (affsItems map[string]interface{}, err error)
- func (j *DSGitHub) AllRoles(ctx *Ctx, rich map[string]interface{}) (roles []string, static bool)
- func (j *DSGitHub) CalculateTimeToReset(ctx *Ctx, rateLimit, rateLimitReset int) (seconds int)
- func (j *DSGitHub) Categories() map[string]struct{}
- func (j *DSGitHub) CustomEnrich() bool
- func (j *DSGitHub) CustomFetchRaw() bool
- func (j *DSGitHub) DateField(*Ctx) string
- func (j *DSGitHub) ElasticRawMapping() []byte
- func (j *DSGitHub) ElasticRichMapping() []byte
- func (j *DSGitHub) Enrich(ctx *Ctx) (err error)
- func (j *DSGitHub) EnrichIssueAssignees(ctx *Ctx, issue map[string]interface{}, assignees []map[string]interface{}, ...) (richItems []interface{}, err error)
- func (j *DSGitHub) EnrichIssueComments(ctx *Ctx, issue map[string]interface{}, comments []map[string]interface{}, ...) (richItems []interface{}, err error)
- func (j *DSGitHub) EnrichIssueItem(ctx *Ctx, item map[string]interface{}, author string, affs bool, ...) (rich map[string]interface{}, err error)
- func (j *DSGitHub) EnrichIssueReactions(ctx *Ctx, issue map[string]interface{}, reactions []map[string]interface{}, ...) (richItems []interface{}, err error)
- func (j *DSGitHub) EnrichItem(ctx *Ctx, item map[string]interface{}, author string, affs bool, ...) (rich map[string]interface{}, err error)
- func (j *DSGitHub) EnrichItems(ctx *Ctx) (err error)
- func (j *DSGitHub) EnrichPullRequestAssignees(ctx *Ctx, pull map[string]interface{}, assignees []map[string]interface{}, ...) (richItems []interface{}, err error)
- func (j *DSGitHub) EnrichPullRequestComments(ctx *Ctx, pull map[string]interface{}, comments []map[string]interface{}, ...) (richItems []interface{}, err error)
- func (j *DSGitHub) EnrichPullRequestItem(ctx *Ctx, item map[string]interface{}, author string, affs bool, ...) (rich map[string]interface{}, err error)
- func (j *DSGitHub) EnrichPullRequestReactions(ctx *Ctx, pull map[string]interface{}, reactions []map[string]interface{}, ...) (richItems []interface{}, err error)
- func (j *DSGitHub) EnrichPullRequestRequestedReviewers(ctx *Ctx, pull map[string]interface{}, ...) (richItems []interface{}, err error)
- func (j *DSGitHub) EnrichPullRequestReviews(ctx *Ctx, pull map[string]interface{}, reviews []map[string]interface{}, ...) (richItems []interface{}, err error)
- func (j *DSGitHub) EnrichRepositoryItem(ctx *Ctx, item map[string]interface{}, author string, affs bool, ...) (rich map[string]interface{}, err error)
- func (j *DSGitHub) FetchItems(ctx *Ctx) (err error)
- func (j *DSGitHub) FetchItemsIssue(ctx *Ctx) (err error)
- func (j *DSGitHub) FetchItemsPullRequest(ctx *Ctx) (err error)
- func (j *DSGitHub) FetchItemsRepository(ctx *Ctx) (err error)
- func (j *DSGitHub) FetchRaw(ctx *Ctx) (err error)
- func (j *DSGitHub) GetFirstIssueAttention(issue map[string]interface{}) (dt time.Time)
- func (j *DSGitHub) GetFirstPullRequestReviewDate(pull map[string]interface{}, commsAndReviews bool) (dt time.Time)
- func (j *DSGitHub) GetItemIdentities(ctx *Ctx, doc interface{}) (identities map[[3]string]struct{}, err error)
- func (j *DSGitHub) GetRoleIdentity(ctx *Ctx, item map[string]interface{}, role string) (identity map[string]interface{})
- func (j *DSGitHub) GitHubIssueEnrichItemsFunc(ctx *Ctx, thrN int, items []interface{}, docs *[]interface{}) (err error)
- func (j *DSGitHub) GitHubPullRequestEnrichItemsFunc(ctx *Ctx, thrN int, items []interface{}, docs *[]interface{}) (err error)
- func (j *DSGitHub) GitHubRepositoryEnrichItemsFunc(ctx *Ctx, thrN int, items []interface{}, docs *[]interface{}) (err error)
- func (j *DSGitHub) HasIdentities() bool
- func (j *DSGitHub) IdentityForObject(ctx *Ctx, item map[string]interface{}) (identity [3]string)
- func (j DSGitHub) Info() string
- func (j *DSGitHub) ItemCategory(item interface{}) string
- func (j *DSGitHub) ItemID(item interface{}) string
- func (j *DSGitHub) ItemUpdatedOn(item interface{}) time.Time
- func (j *DSGitHub) Name() string
- func (j *DSGitHub) OffsetField(*Ctx) string
- func (j *DSGitHub) Origin(ctx *Ctx) string
- func (j *DSGitHub) OriginField(ctx *Ctx) string
- func (j *DSGitHub) ParseArgs(ctx *Ctx) (err error)
- func (j *DSGitHub) ProcessIssue(ctx *Ctx, inIssue map[string]interface{}) (issue map[string]interface{}, err error)
- func (j *DSGitHub) ProcessPull(ctx *Ctx, inPull map[string]interface{}) (pull map[string]interface{}, err error)
- func (j *DSGitHub) ResumeNeedsCategory(ctx *Ctx, raw bool) bool
- func (j *DSGitHub) ResumeNeedsOrigin(ctx *Ctx, raw bool) bool
- func (j *DSGitHub) RichAuthorField(*Ctx) string
- func (j *DSGitHub) RichIDField(*Ctx) string
- func (j *DSGitHub) SupportDateFrom() bool
- func (j *DSGitHub) SupportOffsetFrom() bool
- func (j *DSGitHub) UseDefaultMapping(ctx *Ctx, raw bool) bool
- func (j *DSGitHub) Validate(ctx *Ctx) (err error)
type DSGroupsio
- func (j *DSGroupsio) AddMetadata(ctx *Ctx, msg interface{}) (mItem map[string]interface{})
- func (j *DSGroupsio) AffsItems(ctx *Ctx, rawItem map[string]interface{}, roles []string, date interface{}) (affsItems map[string]interface{}, err error)
- func (j *DSGroupsio) AllRoles(ctx *Ctx, rich map[string]interface{}) (roles []string, static bool)
- func (j *DSGroupsio) CalculateTimeToReset(ctx *Ctx, rateLimit, rateLimitReset int) (seconds int)
- func (j *DSGroupsio) Categories() map[string]struct{}
- func (j *DSGroupsio) CustomEnrich() bool
- func (j *DSGroupsio) CustomFetchRaw() bool
- func (j *DSGroupsio) DateField(*Ctx) string
- func (j *DSGroupsio) ElasticRawMapping() []byte
- func (j *DSGroupsio) ElasticRichMapping() []byte
- func (j *DSGroupsio) Enrich(ctx *Ctx) (err error)
- func (j *DSGroupsio) EnrichItem(ctx *Ctx, item map[string]interface{}, role string, affs bool, ...) (rich map[string]interface{}, err error)
- func (j *DSGroupsio) EnrichItems(ctx *Ctx) (err error)
- func (j *DSGroupsio) FetchItems(ctx *Ctx) (err error)
- func (j *DSGroupsio) FetchRaw(ctx *Ctx) (err error)
- func (j *DSGroupsio) GetItemIdentities(ctx *Ctx, doc interface{}) (identities map[[3]string]struct{}, err error)
- func (j *DSGroupsio) GetItemIdentitiesEx(ctx *Ctx, doc interface{}) (identities map[[3]string]map[string]struct{}, nRecipients int)
- func (j *DSGroupsio) GetRoleIdentity(ctx *Ctx, item map[string]interface{}, role string) map[string]interface{}
- func (j *DSGroupsio) HasIdentities() bool
- func (j DSGroupsio) Info() string
- func (j *DSGroupsio) ItemCategory(item interface{}) string
- func (j *DSGroupsio) ItemID(item interface{}) string
- func (j *DSGroupsio) ItemUpdatedOn(item interface{}) time.Time
- func (j *DSGroupsio) Name() string
- func (j *DSGroupsio) OffsetField(*Ctx) string
- func (j *DSGroupsio) Origin(ctx *Ctx) string
- func (j *DSGroupsio) OriginField(ctx *Ctx) string
- func (j *DSGroupsio) ParseArgs(ctx *Ctx) (err error)
- func (j *DSGroupsio) ResumeNeedsCategory(ctx *Ctx, raw bool) bool
- func (j *DSGroupsio) ResumeNeedsOrigin(ctx *Ctx, raw bool) bool
- func (j *DSGroupsio) RichAuthorField(*Ctx) string
- func (j *DSGroupsio) RichIDField(*Ctx) string
- func (j *DSGroupsio) SupportDateFrom() bool
- func (j *DSGroupsio) SupportOffsetFrom() bool
- func (j *DSGroupsio) UseDefaultMapping(ctx *Ctx, raw bool) bool
- func (j *DSGroupsio) Validate(ctx *Ctx) (err error)
type DSJira
- func (j *DSJira) AddMetadata(ctx *Ctx, issue interface{}) (mItem map[string]interface{})
- func (j *DSJira) AffsItems(ctx *Ctx, item map[string]interface{}, roles []string, date interface{}) (affsItems map[string]interface{}, err error)
- func (j *DSJira) AllRoles(ctx *Ctx, item map[string]interface{}) ([]string, bool)
- func (j *DSJira) CalculateTimeToReset(ctx *Ctx, rateLimit, rateLimitReset int) (seconds int)
- func (j *DSJira) Categories() map[string]struct{}
- func (j *DSJira) CustomEnrich() bool
- func (j *DSJira) CustomFetchRaw() bool
- func (j *DSJira) DateField(*Ctx) string
- func (j *DSJira) ElasticRawMapping() []byte
- func (j *DSJira) ElasticRichMapping() []byte
- func (j *DSJira) Enrich(ctx *Ctx) (err error)
- func (j *DSJira) EnrichItem(ctx *Ctx, item map[string]interface{}, author string, affs bool, ...) (rich map[string]interface{}, err error)
- func (j *DSJira) EnrichItems(ctx *Ctx) (err error)
- func (j *DSJira) FetchItems(ctx *Ctx) (err error)
- func (j *DSJira) FetchRaw(ctx *Ctx) (err error)
- func (j *DSJira) GenSearchFields(ctx *Ctx, issue interface{}, uuid string) (fields map[string]interface{})
- func (j *DSJira) GetFields(ctx *Ctx) (customFields map[string]JiraField, err error)
- func (j *DSJira) GetItemIdentities(ctx *Ctx, doc interface{}) (identities map[[3]string]struct{}, err error)
- func (j *DSJira) GetRoleIdentity(ctx *Ctx, item map[string]interface{}, role string) (identity map[string]interface{})
- func (j *DSJira) HasIdentities() bool
- func (j DSJira) Info() string
- func (j *DSJira) ItemCategory(item interface{}) string
- func (j *DSJira) ItemID(item interface{}) string
- func (j *DSJira) ItemUpdatedOn(item interface{}) time.Time
- func (j *DSJira) Name() string
- func (j *DSJira) OffsetField(*Ctx) string
- func (j *DSJira) Origin(ctx *Ctx) string
- func (j *DSJira) OriginField(ctx *Ctx) string
- func (j *DSJira) ParseArgs(ctx *Ctx) (err error)
- func (j *DSJira) ProcessIssue(ctx *Ctx, allIssues *[]interface{}, allIssuesMtx *sync.Mutex, ...) (wch chan error, err error)
- func (j *DSJira) ResumeNeedsCategory(ctx *Ctx, raw bool) bool
- func (j *DSJira) ResumeNeedsOrigin(ctx *Ctx, raw bool) bool
- func (j *DSJira) RichAuthorField(*Ctx) string
- func (j *DSJira) RichIDField(*Ctx) string
- func (j *DSJira) SearchFields() map[string][]string
- func (j *DSJira) SupportDateFrom() bool
- func (j *DSJira) SupportOffsetFrom() bool
- func (j *DSJira) UseDefaultMapping(ctx *Ctx, raw bool) bool
- func (j *DSJira) Validate(ctx *Ctx) (err error)
type DSRocketchat
- func (j *DSRocketchat) AddMetadata(ctx *Ctx, item interface{}) (mItem map[string]interface{})
- func (j *DSRocketchat) AffsItems(ctx *Ctx, message map[string]interface{}, roles []string, date interface{}) (affsItems map[string]interface{}, err error)
- func (j *DSRocketchat) AllRoles(ctx *Ctx, item map[string]interface{}) ([]string, bool)
- func (j *DSRocketchat) CalculateTimeToReset(ctx *Ctx, rateLimit, rateLimitReset int) (seconds int)
- func (j *DSRocketchat) Categories() map[string]struct{}
- func (j *DSRocketchat) CustomEnrich() bool
- func (j *DSRocketchat) CustomFetchRaw() bool
- func (j *DSRocketchat) DateField(*Ctx) string
- func (j *DSRocketchat) ElasticRawMapping() []byte
- func (j *DSRocketchat) ElasticRichMapping() []byte
- func (j *DSRocketchat) Enrich(ctx *Ctx) (err error)
- func (j *DSRocketchat) EnrichItem(ctx *Ctx, item map[string]interface{}, author string, affs bool, ...) (rich map[string]interface{}, err error)
- func (j *DSRocketchat) EnrichItems(ctx *Ctx) (err error)
- func (j *DSRocketchat) FetchItems(ctx *Ctx) (err error)
- func (j *DSRocketchat) FetchRaw(ctx *Ctx) (err error)
- func (j *DSRocketchat) GetItemIdentities(ctx *Ctx, doc interface{}) (identities map[[3]string]struct{}, err error)
- func (j *DSRocketchat) GetMentions(mentions []interface{}) (richMentions []map[string]interface{})
- func (j *DSRocketchat) GetReactions(reactions map[string]interface{}) (richReactions []map[string]interface{}, nReactions int)
- func (j *DSRocketchat) GetRocketchatMessages(ctx *Ctx, fromDate string, offset, rateLimit, rateLimitReset, thrN int) (messages []map[string]interface{}, ...)
- func (j *DSRocketchat) GetRoleIdentity(ctx *Ctx, item map[string]interface{}, role string) (identity map[string]interface{})
- func (j *DSRocketchat) HasIdentities() bool
- func (j DSRocketchat) Info() string
- func (j *DSRocketchat) ItemCategory(item interface{}) string
- func (j *DSRocketchat) ItemID(item interface{}) string
- func (j *DSRocketchat) ItemUpdatedOn(item interface{}) time.Time
- func (j *DSRocketchat) Name() string
- func (j *DSRocketchat) OffsetField(*Ctx) string
- func (j *DSRocketchat) Origin(ctx *Ctx) string
- func (j *DSRocketchat) OriginField(ctx *Ctx) string
- func (j *DSRocketchat) ParseArgs(ctx *Ctx) (err error)
- func (j *DSRocketchat) ResumeNeedsCategory(ctx *Ctx, raw bool) bool
- func (j *DSRocketchat) ResumeNeedsOrigin(ctx *Ctx, raw bool) bool
- func (j *DSRocketchat) RichAuthorField(*Ctx) string
- func (j *DSRocketchat) RichIDField(*Ctx) string
- func (j *DSRocketchat) SetChannelInfo(rich, channel map[string]interface{})
- func (j *DSRocketchat) SleepAsRequested(res interface{}, thrN int)
- func (j *DSRocketchat) SupportDateFrom() bool
- func (j *DSRocketchat) SupportOffsetFrom() bool
- func (j *DSRocketchat) UseDefaultMapping(ctx *Ctx, raw bool) bool
- func (j *DSRocketchat) Validate(ctx *Ctx) (err error)
type DSStub
- func (j *DSStub) AddMetadata(ctx *Ctx, item interface{}) (mItem map[string]interface{})
- func (j *DSStub) AffsItems(ctx *Ctx, rawItem map[string]interface{}, roles []string, date interface{}) (affsItems map[string]interface{}, err error)
- func (j *DSStub) AllRoles(ctx *Ctx, item map[string]interface{}) ([]string, bool)
- func (j *DSStub) CalculateTimeToReset(ctx *Ctx, rateLimit, rateLimitReset int) (seconds int)
- func (j *DSStub) Categories() map[string]struct{}
- func (j *DSStub) CustomEnrich() bool
- func (j *DSStub) CustomFetchRaw() bool
- func (j *DSStub) DateField(*Ctx) string
- func (j *DSStub) ElasticRawMapping() []byte
- func (j *DSStub) ElasticRichMapping() []byte
- func (j *DSStub) Enrich(ctx *Ctx) (err error)
- func (j *DSStub) EnrichItem(ctx *Ctx, item map[string]interface{}, author string, affs bool, ...) (rich map[string]interface{}, err error)
- func (j *DSStub) EnrichItems(ctx *Ctx) (err error)
- func (j *DSStub) FetchItems(ctx *Ctx) (err error)
- func (j *DSStub) FetchRaw(ctx *Ctx) (err error)
- func (j *DSStub) GetItemIdentities(ctx *Ctx, doc interface{}) (map[[3]string]struct{}, error)
- func (j *DSStub) GetRoleIdentity(ctx *Ctx, item map[string]interface{}, role string) map[string]interface{}
- func (j *DSStub) HasIdentities() bool
- func (j DSStub) Info() string
- func (j *DSStub) ItemCategory(item interface{}) string
- func (j *DSStub) ItemID(item interface{}) string
- func (j *DSStub) ItemUpdatedOn(item interface{}) time.Time
- func (j *DSStub) Name() string
- func (j *DSStub) OffsetField(*Ctx) string
- func (j *DSStub) Origin(ctx *Ctx) string
- func (j *DSStub) OriginField(ctx *Ctx) string
- func (j *DSStub) ParseArgs(ctx *Ctx) (err error)
- func (j *DSStub) ResumeNeedsCategory(ctx *Ctx, raw bool) bool
- func (j *DSStub) ResumeNeedsOrigin(ctx *Ctx, raw bool) bool
- func (j *DSStub) RichAuthorField(*Ctx) string
- func (j *DSStub) RichIDField(*Ctx) string
- func (j *DSStub) SupportDateFrom() bool
- func (j *DSStub) SupportOffsetFrom() bool
- func (j *DSStub) UseDefaultMapping(ctx *Ctx, raw bool) bool
- func (j *DSStub) Validate(ctx *Ctx) (err error)
type DateCacheEntry
type ESCacheEntry
- func ESCacheGet(ctx *Ctx, key string) (entry *ESCacheEntry, ok bool)
type Flag
- func NewFlag() *Flag
- func (f *Flag) Bool() bool
- func (f *Flag) Date() *time.Time
- func (f *Flag) Int() int
- func (f *Flag) Set(val string) error
- func (f *Flag) String() string
type GoogleGroups
type JiraField
type MemCacheEntry
type PLS
type PiperMail
type RawPLS
type Repository

Constants ¶

View Source

const (
	// BulkRefreshMode - bulk upload refresh mode, can be: false, true, wait_for (ES defaults to false)
	BulkRefreshMode = "true"
	// BulkWaitForActiveShardsMode - bulk upload wait_for_active_shards mode, can be: 1, 2, ..., all (ES defaults to 1)
	BulkWaitForActiveShardsMode = "all"
	// KeywordMaxlength - max description length
	KeywordMaxlength = 1000
	// DefaultRateLimitHeader - default value for rate limit header
	DefaultRateLimitHeader = "X-RateLimit-Remaining"
	// DefaultRateLimitResetHeader - default value for rate limit reset header
	DefaultRateLimitResetHeader = "X-RateLimit-Reset"
)

View Source

const (
	// GerritBackendVersion - backend version
	GerritBackendVersion = "0.1.1"
	// GerritDefaultSSHKeyPath - default path to look for gerrit ssh private key
	GerritDefaultSSHKeyPath = "$HOME/.ssh/id_rsa"
	// GerritDefaultSSHPort - default gerrit ssh port
	GerritDefaultSSHPort = 29418
	// GerritDefaultMaxReviews = default max reviews when processing gerrit
	GerritDefaultMaxReviews = 1000
	// GerritCodeReviewApprovalType - code review approval type
	GerritCodeReviewApprovalType = "Code-Review"
)

View Source

const (
	// GitBackendVersion - backend version
	GitBackendVersion = "0.1.1"
	// GitDefaultReposPath - default path where git repository clones
	GitDefaultReposPath = "$HOME/.perceval/repositories"
	// GitDefaultCachePath - default path where gitops cache files are stored
	GitDefaultCachePath = "$HOME/.perceval/cache"
	// GitOpsCommand - command that maintains git stats cache
	// GitOpsCommand = "gitops.py"
	GitOpsCommand = "gitops"
	// GitOpsFailureFatal - is GitOpsCommand failure fatal?
	GitOpsFailureFatal = false
	// OrphanedCommitsCommand - command to list orphaned commits
	OrphanedCommitsCommand = "detect-removed-commits.sh"
	// OrphanedCommitsFailureFatal - is OrphanedCommitsCommand failure fatal?
	OrphanedCommitsFailureFatal = true
	// GitOpsNoCleanup - if set, it will skip gitops repo cleanup
	GitOpsNoCleanup = false
	// GitParseStateInit - init parser state
	GitParseStateInit = 0
	// GitParseStateCommit - commit parser state
	GitParseStateCommit = 1
	// GitParseStateHeader - header parser state
	GitParseStateHeader = 2
	// GitParseStateMessage - message parser state
	GitParseStateMessage = 3
	// GitParseStateFile - file parser state
	GitParseStateFile = 4
	// GitCommitDateField - date field in the commit structure
	GitCommitDateField = "CommitDate"
	// GitDefaultSearchField - default search field
	GitDefaultSearchField = "item_id"
	// GitUUID - field used as a rich item ID when pair progrmamming is enabled
	GitUUID = "git_uuid"
	// GitHubURL - GitHub URL
	GitHubURL = "https://github.com/"
	// GitMaxCommitProperties - maximum properties that can be set on the commit object
	GitMaxCommitProperties = 300
	// GitGenerateFlatDocs - do we want to generate flat commit co-authors docs, like docs with type: commit_co_author, commit_signer etc.?
	GitGenerateFlatDocs = true
)

View Source

const (
	// GitHubBackendVersion - backend version
	GitHubBackendVersion = "0.1.0"
	// GitHubURLRoot - GitHub URL root
	GitHubURLRoot = "https://github.com/"
	// MaxGitHubUsersFileCacheAge 90 days (in seconds) - file is considered too old anywhere between 90-180 days
	MaxGitHubUsersFileCacheAge = 7776000
	// MaxCommentBodyLength - max comment body length
	MaxCommentBodyLength = 4096
	// MaxIssueBodyLength - max issue body length
	MaxIssueBodyLength = 4096
	// MaxPullBodyLength - max pull request body length
	MaxPullBodyLength = 4096
	// MaxReviewBodyLength - max review body length
	MaxReviewBodyLength = 4096
	// MaxReviewCommentBodyLength - max review comment body length
	MaxReviewCommentBodyLength = 4096
	// ItemsPerPage - how many items in a page
	ItemsPerPage = 100
	// AbuseWaitSeconds - N - wait random(N:2N) seconds if GitHub detected abuse
	// 7 means from 7 to 13 seconds, 10 on average
	AbuseWaitSeconds = 7
	// CacheGitHubRepo - cache this?
	CacheGitHubRepo = true
	// CacheGitHubIssues - cache this?
	CacheGitHubIssues = false
	// CacheGitHubUser - cache this?
	CacheGitHubUser = true
	// CacheGitHubUserFiles - cache this in files?
	CacheGitHubUserFiles = true
	// CacheGitHubIssueComments - cache this?
	CacheGitHubIssueComments = false
	// CacheGitHubCommentReactions - cache this?
	CacheGitHubCommentReactions = false
	// CacheGitHubIssueReactions - cache this?
	CacheGitHubIssueReactions = false
	// CacheGitHubPull - cache this?
	CacheGitHubPull = false
	// CacheGitHubPulls - cache this?
	CacheGitHubPulls = false
	// CacheGitHubPullReviews - cache this?
	CacheGitHubPullReviews = false
	// CacheGitHubPullReviewComments - cache this?
	CacheGitHubPullReviewComments = false
	// CacheGitHubReviewCommentReactions - cache this?
	CacheGitHubReviewCommentReactions = false
	// CacheGitHubPullRequestedReviewers - cache this?
	CacheGitHubPullRequestedReviewers = false
	// CacheGitHubPullCommits - cache this?
	CacheGitHubPullCommits = false
	// CacheGitHubUserOrgs - cache this?
	CacheGitHubUserOrgs = true
	// WantEnrichIssueAssignees - do we want to create rich documents for issue assignees (it contains identity data too).
	WantEnrichIssueAssignees = true
	// WantEnrichIssueCommentReactions - do we want to create rich documents for issue comment reactions (it contains identity data too).
	WantEnrichIssueCommentReactions = true
	// WantEnrichIssueReactions - do we want to create rich documents for issue reactions (it contains identity data too).
	WantEnrichIssueReactions = true
	// WantEnrichPullRequestAssignees - do we want to create rich documents for pull request assignees (it contains identity data too).
	WantEnrichPullRequestAssignees = true
	// WantEnrichPullRequestCommentReactions - do we want to create rich documents for pull request comment reactions (it contains identity data too).
	WantEnrichPullRequestCommentReactions = true
	// WantEnrichPullRequestRequestedReviewers - do we want to create rich documents for pull request requested reviewers (it contains identity data too).
	WantEnrichPullRequestRequestedReviewers = true
)

View Source

const (
	// GroupsioBackendVersion - backend version
	GroupsioBackendVersion = "0.1.3"
	// GroupsioURLRoot - root url for group name origin
	GroupsioURLRoot = "https://groups.io/g/"
	// GroupsioAPIURL - Groups.io API URL
	GroupsioAPIURL = "https://groups.io/api/v1"
	// GroupsioAPILogin - login API
	GroupsioAPILogin = "/login"
	// GroupsioAPIGetsubs - getsubs API
	GroupsioAPIGetsubs = "/getsubs"
	// GroupsioAPIDownloadArchives - download archives API
	GroupsioAPIDownloadArchives = "/downloadarchives"
	// GroupsioDefaultArchPath - default path where archives are stored
	GroupsioDefaultArchPath = "$HOME/.perceval/mailinglists"
	// GroupsioMBoxFile - default messages file name
	GroupsioMBoxFile = "messages.zip"
	// GroupsioMessageIDField - message ID field from email
	GroupsioMessageIDField = "message-id"
	// GroupsioMessageDateField - message ID field from email
	GroupsioMessageDateField = "date"
	// GroupsioMessageReceivedField - message Received filed
	GroupsioMessageReceivedField = "received"
	// GroupsioDefaultSearchField - default search field
	GroupsioDefaultSearchField = "item_id"
	// GroupsioMaxMessageBodyLength - trucacte message bodies longer than this (per each multi-body email part)
	GroupsioMaxMessageBodyLength = 1000
	// GroupsioMaxRichMessageLines - maximum numbe rof message text/plain lines copied to rich index
	GroupsioMaxRichMessageLines = 10
	// GroupsioMaxRecipients - maximum number of emails parsed from To:
	GroupsioMaxRecipients = 50
	// GroupsioMaxMessageProperties - maximum properties that can be set on the message object
	GroupsioMaxMessageProperties = 255
)

View Source

const (
	// JiraAPIRoot - main API path
	JiraAPIRoot = "/rest/api/2"
	// JiraAPISearch - search API subpath
	JiraAPISearch = "/search"
	// JiraAPIField - field API subpath
	JiraAPIField = "/field"
	// JiraAPIIssue - issue API subpath
	JiraAPIIssue = "/issue"
	// JiraAPIComment - comments API subpath
	JiraAPIComment = "/comment"
	// JiraBackendVersion - backend version
	JiraBackendVersion = "0.1.1"
	// JiraDefaultSearchField - default search field
	JiraDefaultSearchField = "item_id"
	// JiraFilterByProjectInComments - filter by project when searching for comments
	JiraFilterByProjectInComments = false
	// JiraDropCustomFields - drop custom fields from raw index
	JiraDropCustomFields = true
	// JiraMapCustomFields - run custom fields mapping
	JiraMapCustomFields = true
	// ClosedStatusCategoryKey - issue closed status key
	ClosedStatusCategoryKey = "done"
	// JiraRichAuthorField - rich index author field
	JiraRichAuthorField = "reporter"
)

View Source

const Approval = "approval"

Approval - common constant string

View Source

const Author string = "author"

Author - default author field

View Source

const CacheCleanupProb int = 2

CacheCleanupProb - probability of cache cleanup in %

View Source

const Changeset = "changeset"

Changeset - common constant string

View Source

const Comment string = "comment"

Comment - common constant string

View Source

const Commit = "commit"

Commit - common constant string

View Source

const Confluence string = "confluence"

Confluence - common constant string

View Source

const (
	// ConfluenceBackendVersion - backend version
	ConfluenceBackendVersion = "0.1.0"
)

View Source

const ContentType = "Content-Type"

ContentType - common constant string

View Source

const DADSOrigin string = "dads"

DADSOrigin - common constant string

View Source

const DadsWarning = "da-ds WARNING"

DadsWarning - common constant string

View Source

const DefaultAuthorField string = "author"

DefaultAuthorField - default author field

View Source

const DefaultDateField string = "metadata__updated_on"

DefaultDateField - default date field

View Source

const DefaultEnrichDateField string = "metadata__enriched_on"

DefaultEnrichDateField - default date field

View Source

const DefaultIDField string = "id"

DefaultIDField - default id field

View Source

const DefaultOffsetField string = "updated_on"

DefaultOffsetField - default offset field

View Source

const DefaultOriginField string = "origin"

DefaultOriginField - default origin field

View Source

const DefaultTagField string = "tag"

DefaultTagField - default tag field

View Source

const DefaultTimestampField string = "metadata__timestamp"

DefaultTimestampField - default timestamp field

View Source

const Delete string = "DELETE"

Delete - common constant string

View Source

const From = "from"

From - common constant string

View Source

const Gerrit string = "gerrit"

Gerrit - common constant string

View Source

const Get string = "GET"

Get - common constant string

View Source

const Git string = "git"

Git - common constant string

View Source

const GitHub string = "github"

GitHub - common constant string

View Source

const Groupsio string = "groupsio"

Groupsio - common constant string

View Source

const HistoricalContent = "historical content"

HistoricalContent - common constant string

View Source

const ID = "id"

ID - common constant string

View Source

const Issue string = "issue"

Issue - common constant string

View Source

const Jira string = "jira"

Jira - common constant string

View Source

const LowerContentType = "content-type"

LowerContentType - common constant string

View Source

const (
	// MBoxDropXFields - drop fields starting with X- - to avoid ES 1000 fields limit
	MBoxDropXFields = true
)

View Source

const (
	// MaxPayloadPrintfLen - truncate messages longer than this
	MaxPayloadPrintfLen = 0x2000
)

View Source

const Message = "message"

Message - common constant string

View Source

const MissingName = "-MISSING-NAME"

MissingName - common constant string

View Source

const (
	// MultiOrgNames - suffix for multiple orgs affiliation data
	MultiOrgNames = "_multi_org_names"
)

View Source

const Nil = "none"

Nil - common constant string

View Source

const NoSearchContextFound = "No search context found for id"

NoSearchContextFound - this appearch when processing takes so long, that the scroll expires

View Source

const None = "None"

None - common constant string

View Source

const OK string = "ok"

OK - common constant string

View Source

const Offset = "offset"

Offset - common constant string

View Source

const Patchset = "patchset"

Patchset - common constant string

View Source

const Post string = "POST"

Post - common constant string

View Source

const ProjectSlug = "project_slug"

ProjectSlug - common constant string

View Source

const Put string = "PUT"

Put - common constant string

View Source

const Recipient string = "recipient"

Recipient - default author field

View Source

const Redacted string = "[redacted]"

Redacted - [redacted]

View Source

const RedactedEmail = "-REDACTED-EMAIL"

RedactedEmail - common constant string

View Source

const Review = "review"

Review - common constant string

View Source

const Rocketchat string = "rocketchat"

Rocketchat - common constant string

View Source

const (
	// RocketchatBackendVersion - backend version
	RocketchatBackendVersion = "0.0.1"
)

View Source

const Stub string = "stub"

Stub - common constant string

View Source

const (
	// StubBackendVersion - backend version
	StubBackendVersion = "0.0.0"
)

View Source

const TooManyScrolls = "Trying to create too many scroll contexts"

TooManyScrolls - this appearch in error message when too many scrolls are created

View Source

const UUID = "uuid"

UUID - common constant string

View Source

const Unknown = "Unknown"

Unknown - common constant string

View Source

const Wait59m = "59m"

Wait59m - common constant string

Variables ¶

View Source

var (
	// AffsFields - all properties added by affiliations (excluding multi org name)
	// AffsFields = []string{"_id", "_uuid", "_name", "_user_name", "_domain", "_gender", "_gender_acc", "_org_name", "_bot"}
	AffsFields = []string{"_id", "_uuid", "_name", "_user_name", "_domain", "_org_name", "_bot"}
	// RequiredAffsFields - required affs fields
	RequiredAffsFields = []string{"_org_name", "_name", "_user_name"}
)

View Source

var (
	// SettingsFieldsNumberLimit - make maximum number of index fields bigger (some raw indices have a lot of fields and we don't control this)
	SettingsFieldsNumberLimit = []byte(`{"index.mapping.total_fields.limit":50000}`)
	// MappingNotAnalyzeString - make all string keywords by default (not analyze them)
	MappingNotAnalyzeString = []byte(`{"dynamic_templates":[{"notanalyzed":{"match":"*","match_mapping_type":"string","mapping":{"type":"keyword"}}},{"formatdate":{"match":"*","match_mapping_type":"date","mapping":{"type":"date","format":"strict_date_optional_time||epoch_millis"}}}]}`)
	// RawFields - standard raw fields
	RawFields = []string{DefaultDateField, DefaultTimestampField, DefaultOriginField, DefaultTagField, UUID, Offset}
	// DefaultDateFrom - default date from
	DefaultDateFrom = time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC)
)

View Source

var (
	// ConfluenceRawMapping - Confluence raw index mapping
	ConfluenceRawMapping = []byte(`{"dynamic":true,"properties":{"data":{"properties":{"metadata__updated_on":{"type":"date"},"extensions":{"dynamic":false,"properties":{}},"ancestors":{"properties":{"extensions":{"dynamic":false,"properties":{}}}},"body":{"dynamic":false,"properties":{}}}}}}`)
	// ConfluenceRichMapping - Confluence rich index mapping
	ConfluenceRichMapping = []byte(`{"properties":{"metadata__updated_on":{"type":"date"},"title_analyzed":{"type":"text","index":true}}}`)
	// ConfluenceCategories - categories defined for Confluence
	ConfluenceCategories = map[string]struct{}{HistoricalContent: {}}
	// ConfluenceDefaultMaxContents - max contents to fetch at a time
	ConfluenceDefaultMaxContents = 1000
	// ConfluenceDefaultSearchField - default search field
	ConfluenceDefaultSearchField = "item_id"
	// ConfluenceContentRoles - roles to fetch affiliation data for historical content
	ConfluenceContentRoles = []string{"by"}
	// ConfluenceRichAuthorField - rich index author field
	ConfluenceRichAuthorField = "by"
)

View Source

var (
	// GerritRawMapping - Gerrit raw index mapping
	GerritRawMapping = []byte(`{"dynamic":true,"properties":{"metadata__updated_on":{"type":"date"},"data":{"properties":{"commitMessage":{"type":"text","index":true},"comments":{"properties":{"message":{"type":"text","index":true}}},"subject":{"type":"text","index":true},"patchSets":{"properties":{"approvals":{"properties":{"description":{"type":"text","index":true}}},"comments":{"properties":{"message":{"type":"text","index":true}}}}}}}}}`)
	// GerritRichMapping - Gerrit rich index mapping
	GerritRichMapping = []byte(`{"properties":{"metadata__updated_on":{"type":"date"},"approval_description_analyzed":{"type":"text","index":true},"comment_message_analyzed":{"type":"text","index":true},"status":{"type":"keyword"},"summary_analyzed":{"type":"text","index":true},"timeopen":{"type":"double"}}}`)
	// GerritCategories - categories defined for gerrit
	GerritCategories = map[string]struct{}{Review: {}}
	// GerritVersionRegexp - gerrit verion pattern
	GerritVersionRegexp = regexp.MustCompile(`gerrit version (\d+)\.(\d+).*`)
	// GerritDefaultSearchField - default search field
	GerritDefaultSearchField = "item_id"
	// GerritReviewRoles - roles to fetch affiliation data for review
	GerritReviewRoles = []string{"owner"}
	// GerritCommentRoles - roles to fetch affiliation data for comment
	GerritCommentRoles = []string{"reviewer"}
	// GerritPatchsetRoles - roles to fetch affiliation data for patchset
	GerritPatchsetRoles = []string{"author", "uploader"}
	// GerritApprovalRoles - roles to fetch affiliation data for approval
	GerritApprovalRoles = []string{"by"}
)

View Source

var (
	// GitRawMapping - Git raw index mapping
	GitRawMapping = []byte(`{"dynamic":true,"properties":{"metadata__updated_on":{"type":"date"},"data":{"properties":{"message":{"type":"text","index":true}}}}}`)
	// GitRichMapping - Git rich index mapping
	// GitRichMapping = []byte(`{"dynamic":true,"properties":{"file_data":{"type":"nested"},"author_name":{"type":"keyword"},"metadata__updated_on":{"type":"date","format":"strict_date_optional_time||epoch_millis"},"message_analyzed":{"type":"text","index":true}},"dynamic_templates":[{"notanalyzed":{"match":"*","unmatch":"message_analyzed","match_mapping_type":"string","mapping":{"type":"keyword"}}},{"formatdate":{"match":"*","match_mapping_type":"date","mapping":{"format":"strict_date_optional_time||epoch_millis","type":"date"}}}]}`)
	// GitRichMapping = []byte(`{"dynamic":true,"properties":{"file_data":{"type":"nested"},"authors_signed":{"type":"nested"},"authors_co_authored":{"type":"nested"},"authors_tested":{"type":"nested"},"authors_approved":{"type":"nested"},"authors_reviewed":{"type":"nested"},"authors_reported":{"type":"nested"},"authors_informed":{"type":"nested"},"authors_resolved":{"type":"nested"},"authors_influenced":{"type":"nested"},"author_name":{"type":"keyword"},"pair_programming_commit":{"type":"float"},"time_to_commit_hours":{"type":"float"},"metadata__updated_on":{"type":"date","format":"strict_date_optional_time||epoch_millis"},"message_analyzed":{"type":"text","index":true}},"dynamic_templates":[{"notanalyzed":{"match":"*","unmatch":"message_analyzed","match_mapping_type":"string","mapping":{"type":"keyword"}}},{"formatdate":{"match":"*","match_mapping_type":"date","mapping":{"format":"strict_date_optional_time||epoch_millis","type":"date"}}}]}`)
	GitRichMapping = []byte(`{"dynamic":true,"properties":{"file_data":{"type":"nested"},"authors_signed":{"type":"nested"},"authors_co_authored":{"type":"nested"},"authors_tested":{"type":"nested"},"authors_approved":{"type":"nested"},"authors_reviewed":{"type":"nested"},"authors_reported":{"type":"nested"},"authors_informed":{"type":"nested"},"authors_resolved":{"type":"nested"},"authors_influenced":{"type":"nested"},"author_name":{"type":"keyword"},"pair_programming_commit":{"type":"float"},"metadata__updated_on":{"type":"date","format":"strict_date_optional_time||epoch_millis"},"message_analyzed":{"type":"text","index":true}},"dynamic_templates":[{"notanalyzed":{"match":"*","unmatch":"message_analyzed","match_mapping_type":"string","mapping":{"type":"keyword"}}},{"formatdate":{"match":"*","match_mapping_type":"date","mapping":{"format":"strict_date_optional_time||epoch_millis","type":"date"}}}]}`)
	// GitCategories - categories defined for git
	GitCategories = map[string]struct{}{Commit: {}}
	// GitDefaultEnv - default git command environment
	GitDefaultEnv = map[string]string{"LANG": "C", "PAGER": ""}
	// GitLogOptions - default git log options
	GitLogOptions = []string{
		"--raw",
		"--numstat",
		"--pretty=fuller",
		"--decorate=full",
		"--parents",
		"-M",
		"-C",
		"-c",
	}
	// GitCommitPattern - pattern to match a commit
	GitCommitPattern = regexp.MustCompile(`^commit[ \t](?P<commit>[a-f0-9]{40})(?:[ \t](?P<parents>[a-f0-9][a-f0-9 \t]+))?(?:[ \t]\((?P<refs>.+)\))?$`)
	// GitHeaderPattern - pattern to match a commit
	GitHeaderPattern = regexp.MustCompile(`^(?P<name>[a-zA-z0-9\-]+)\:[ \t]+(?P<value>.+)$`)
	// GitMessagePattern - message patterns
	GitMessagePattern = regexp.MustCompile(`^[\s]{4}(?P<msg>.*)$`)
	// GitTrailerPattern - message trailer pattern
	GitTrailerPattern = regexp.MustCompile(`^(?P<name>[a-zA-z0-9\-]+)\:[ \t]+(?P<value>.+)$`)
	// GitActionPattern - action pattern - note that original used `\.{,3}` which is not supported in go - you must specify from=0: `\.{0,3}`
	GitActionPattern = regexp.MustCompile(`^(?P<sc>\:+)(?P<modes>(?:\d{6}[ \t])+)(?P<indexes>(?:[a-f0-9]+\.{0,3}[ \t])+)(?P<action>[^\t]+)\t+(?P<file>[^\t]+)(?:\t+(?P<newfile>.+))?$`)
	// GitStatsPattern - stats pattern
	GitStatsPattern = regexp.MustCompile(`^(?P<added>\d+|-)[ \t]+(?P<removed>\d+|-)[ \t]+(?P<file>.+)$`)
	// GitAuthorsPattern - author pattern
	GitAuthorsPattern = regexp.MustCompile(`(?P<first_authors>.* .*) and (?P<last_author>.* .*) (?P<email>.*)`)
	// GitCoAuthorsPattern - author pattern
	GitCoAuthorsPattern = regexp.MustCompile(`Co-authored-by:(?P<first_authors>.* .*)<(?P<email>.*)>\n?`)
	// GitDocFilePattern - files matching this pattern are detected as documentation files, so commit will be marked as doc_commit
	GitDocFilePattern = regexp.MustCompile(`(?i)(\.md$|\.rst$|\.docx?$|\.txt$|\.pdf$|\.jpe?g$|\.png$|\.svg$|\.img$|^docs/|^documentation/|^readme)`)
	// GitCommitRoles - roles to fetch affiliation data
	GitCommitRoles = []string{"Author", "Commit"}
	// GitPPAuthors - flag to authors mapping used in pair programming mode
	GitPPAuthors = map[string]string{
		"is_git_commit_multi_author":    "authors",
		"is_git_commit_multi_committer": "committers",
		"is_git_commit_signed_off":      "authors_signed_off",
		"is_git_commit_co_author":       "co_authors",
	}
	// GitTrailerPPAuthors - trailer name to authors map (for pair programming)
	GitTrailerPPAuthors = map[string]string{"Signed-off-by": "authors_signed_off", "Co-authored-by": "co_authors"}
	// GitAllowedTrailers - allowed commit trailer flags (lowercase/case insensitive -> correct case)
	GitAllowedTrailers = map[string][]string{}/* 348 elements not displayed */

	// GitTrailerOtherAuthors - trailer name to authors map (for all documents)
	GitTrailerOtherAuthors = map[string][2]string{
		"Signed-off-by":  {"authors_signed", "signer"},
		"Co-authored-by": {"authors_co_authored", "co_author"},
		"Tested-by":      {"authors_tested", "tester"},
		"Approved-by":    {"authors_approved", "approver"},
		"Reviewed-by":    {"authors_reviewed", "reviewer"},
		"Reported-by":    {"authors_reported", "reporter"},
		"Informed-by":    {"authors_informed", "informer"},
		"Resolved-by":    {"authors_resolved", "resolver"},
		"Influenced-by":  {"authors_influenced", "influencer"},
	}
	// GitTrailerSameAsAuthor - can a given trailer be the same as the main commit's author?
	GitTrailerSameAsAuthor = map[string]bool{
		"Signed-off-by":  true,
		"Co-authored-by": false,
		"Tested-by":      true,
		"Approved-by":    false,
		"Reviewed-by":    false,
		"Reported-by":    true,
		"Informed-by":    true,
		"Resolved-by":    true,
		"Influenced-by":  true,
	}
	// CommitsHash is a map of commit hashes for each repo
	CommitsHash = make(map[string]map[string]struct{})
)

View Source

var (
	// GitHubRawMapping - GitHub raw index mapping
	// GitHubRawMapping = []byte(`{"properties":{"metadata__updated_on":{"type":"date"}}}`)
	GitHubRawMapping = []byte(`{"dynamic":true,"properties":{"metadata__updated_on":{"type":"date","format":"strict_date_optional_time||epoch_millis"},"data":{"properties":{"comments_data":{"dynamic":false,"properties":{"body":{"type":"text","index":true}}},"review_comments_data":{"dynamic":false,"properties":{"body":{"type":"text","index":true},"diff_hunk":{"type":"text","index":true}}},"reviews_data":{"dynamic":false,"properties":{"body":{"type":"text","index":true}}},"body":{"type":"text","index":true}}}}}`)
	// GitHubRichMapping - GitHub rich index mapping
	// GitHubRichMapping = []byte(`{"properties":{"metadata__updated_on":{"type":"date"},"merge_author_geolocation":{"type":"geo_point"},"assignee_geolocation":{"type":"geo_point"},"state":{"type":"keyword"},"user_geolocation":{"type":"geo_point"},"title_analyzed":{"type":"text","index":true}}}`)
	// GitHubRichMapping = []byte(`{"dynamic":true,"properties":{"metadata__updated_on":{"type":"date","format":"strict_date_optional_time||epoch_millis"},"merge_author_geolocation":{"type":"geo_point"},"assignee_geolocation":{"type":"geo_point"},"id_in_repo":{"type":"long"},"state":{"type":"keyword"},"user_geolocation":{"type":"geo_point"},"title_analyzed":{"type":"text","index":true},"body_analyzed":{"type":"text","index":true},"code_merge_duration":{"type":"float"},"time_open_days":{"type":"float"},"time_to_close_days":{"type":"float"},"time_to_first_attention":{"type":"float"},"time_to_merge_request_response":{"type":"float"},"id_in_repo":{"type":"long"}},"dynamic_templates":[{"notanalyzed":{"match":"*","unmatch":"body","match_mapping_type":"string","mapping":{"type":"keyword"}}},{"formatdate":{"match":"*","match_mapping_type":"date","mapping":{"format":"strict_date_optional_time||epoch_millis","type":"date"}}}]}`)
	GitHubRichMapping = []byte(`{"dynamic":true,"properties":{"metadata__updated_on":{"type":"date","format":"strict_date_optional_time||epoch_millis"},"merge_author_geolocation":{"type":"geo_point"},"assignee_geolocation":{"type":"geo_point"},"state":{"type":"keyword"},"user_geolocation":{"type":"geo_point"},"title_analyzed":{"type":"text","index":true},"body_analyzed":{"type":"text","index":true},"code_merge_duration":{"type":"float"},"time_open_days":{"type":"float"},"time_to_close_days":{"type":"float"},"time_to_first_attention":{"type":"float"},"time_to_merge_request_response":{"type":"float"},"id_in_repo":{"type":"long"}},"dynamic_templates":[{"notanalyzed":{"match":"*","unmatch":"body","match_mapping_type":"string","mapping":{"type":"keyword"}}},{"formatdate":{"match":"*","match_mapping_type":"date","mapping":{"format":"strict_date_optional_time||epoch_millis","type":"date"}}}]}`)
	// GitHubCategories - categories defined for GitHub
	GitHubCategories = map[string]struct{}{"issue": {}, "pull_request": {}, "repository": {}}
	// GitHubIssueRoles - roles to fetch affiliation data for github issue
	GitHubIssueRoles = []string{"user_data", "assignee_data"}
	// GitHubIssueCommentRoles - roles to fetch affiliation data for github issue comment
	GitHubIssueCommentRoles = []string{"user_data"}
	// GitHubIssueAssigneeRoles - roles to fetch affiliation data for github issue comment
	GitHubIssueAssigneeRoles = []string{"assignee"}
	// GitHubIssueReactionRoles - roles to fetch affiliation data for github issue reactions or issue comment reactions
	GitHubIssueReactionRoles = []string{"user_data"}
	// GitHubPullRequestRoles - roles to fetch affiliation data for github pull request
	GitHubPullRequestRoles = []string{"user_data", "assignee_data", "merged_by_data"}
	// GitHubPullRequestCommentRoles - roles to fetch affiliation data for github pull request comment
	GitHubPullRequestCommentRoles = []string{"user_data"}
	// GitHubPullRequestAssigneeRoles - roles to fetch affiliation data for github pull request comment
	GitHubPullRequestAssigneeRoles = []string{"assignee"}
	// GitHubPullRequestReactionRoles - roles to fetch affiliation data for github pull request comment reactions
	GitHubPullRequestReactionRoles = []string{"user_data"}
	// GitHubPullRequestRequestedReviewerRoles - roles to fetch affiliation data for github pull request requested reviewer
	GitHubPullRequestRequestedReviewerRoles = []string{"requested_reviewer"}
	// GitHubPullRequestReviewRoles - roles to fetch affiliation data for github pull request comment
	GitHubPullRequestReviewRoles = []string{"user_data"}
)

View Source

var (
	// GroupsioRawMapping - Groupsio raw index mapping
	GroupsioRawMapping = []byte(`{"dynamic":true,"properties":{"metadata__updated_on":{"type":"date"},"data":{"properties":{"body":{"dynamic":false,"properties":{}}}}}}`)
	// GroupsioRichMapping - Groupsio rich index mapping
	GroupsioRichMapping = []byte(`{"properties":{"metadata__updated_on":{"type":"date"},"Subject_analyzed":{"type":"text","fielddata":true,"index":true},"body":{"type":"text","index":true}}}`)
	// GroupsioCategories - categories defined for Groupsio
	GroupsioCategories = map[string]struct{}{Message: {}}
	// GroupsioMBoxMsgSeparator - used to split mbox file into separate messages
	GroupsioMBoxMsgSeparator = []byte("\nFrom ")
	// GroupsioMsgLineSeparator - used to split mbox message into its separate lines
	GroupsioMsgLineSeparator = []byte("\r\n")
)

View Source

var (
	// JiraSearchFields - extra search fields
	JiraSearchFields = map[string][]string{
		"project_id":   {"fields", "project", "id"},
		"project_key":  {"fields", "project", "key"},
		"project_name": {"fields", "project", "name"},
		"issue_key":    {"key"},
	}
	// JiraRawMapping - Jira raw index mapping
	// JiraRawMapping = []byte(`{"dynamic":true,"properties":{"metadata__updated_on":{"type":"date"},"data":{"properties":{"renderedFields":{"dynamic":false,"properties":{}},"operations":{"dynamic":false,"properties":{}},"fields":{"dynamic":true,"properties":{"description":{"type":"text","index":true},"workratio":{"type":"double"},"environment":{"type":"text","index":true}}},"changelog":{"properties":{"histories":{"dynamic":false,"properties":{}}}},"comments_data":{"properties":{"body":{"type":"text","index":true}}}}}}}`)
	JiraRawMapping = []byte(`{"properties":{"metadata__updated_on":{"type":"date","format":"strict_date_optional_time||epoch_millis"},"main_description_analyzed":{"type":"text","index":true},"releases":{"type":"keyword"},"body":{"type":"text","index":true}},"dynamic_templates":[{"notanalyzed":{"match":"*","unmatch":"body","match_mapping_type":"string","mapping":{"type":"keyword"}}},{"formatdate":{"match":"*","match_mapping_type":"date","mapping":{"format":"strict_date_optional_time||epoch_millis","type":"date"}}}]}`)
	// JiraRichMapping - Jira rich index mapping
	// JiraRichMapping = []byte(`{"properties":{"metadata__updated_on":{"type":"date"},"main_description_analyzed":{"type":"text","index":true},"releases":{"type":"keyword"},"body":{"type":"text","index":true}}}`)
	JiraRichMapping = []byte(`{"dynamic":true,"properties":{"metadata__updated_on":{"type":"date","format":"strict_date_optional_time||epoch_millis"},"data":{"properties":{"renderedFields":{"dynamic":false,"properties":{}},"operations":{"dynamic":false,"properties":{}},"fields":{"dynamic":true,"properties":{"description":{"type":"text","index":true},"environment":{"type":"text","index":true}}},"changelog":{"properties":{"histories":{"dynamic":false,"properties":{}}}},"comments_data":{"properties":{"body":{"type":"text","index":true}}}}}},"dynamic_templates":[{"notanalyzed":{"match":"*","unmatch":"body","match_mapping_type":"string","mapping":{"type":"keyword"}}},{"workratio":{"match":"workratio","match_mapping_type":"long","mapping":{"type":"double"}}},{"formatdate":{"match":"*","match_mapping_type":"date","mapping":{"format":"strict_date_optional_time||epoch_millis","type":"date"}}}]}`)
	// JiraRoles - roles defined for Jira backend
	JiraRoles = []string{"assignee", "reporter", "creator", Author, "updateAuthor"}
	// JiraCategories - categories defined for Jira
	JiraCategories = map[string]struct{}{Issue: {}}
	// JiraKeepCustomFiled - we're dropping all but those custom fields
	JiraKeepCustomFiled = map[string]struct{}{"Story Points": {}, "Sprint": {}}
)

View Source

var (
	// RocketchatRawMapping - Rocketchat raw index mapping
	RocketchatRawMapping = []byte(`{"dynamic":true,"properties":{"metadata__updated_on":{"type":"date"},"data":{"dynamic":false,"properties":{}}}}`)
	// RocketchatRichMapping - Rocketchat rich index mapping
	RocketchatRichMapping = []byte(`{"properties":{"metadata__updated_on":{"type":"date"},"msg_analyzed":{"type":"text","fielddata":true,"index":true}}}`)
	// RocketchatCategories - categories defined for Rocketchat
	RocketchatCategories = map[string]struct{}{Message: {}}
	// RocketchatDefaultMaxItems - max items to retrieve from API via a single request
	RocketchatDefaultMaxItems = 100
	// RocketchatDefaultMinRate - default min rate points (when not set)
	RocketchatDefaultMinRate = 10
	// RocketchatDefaultSearchField - default search field
	RocketchatDefaultSearchField = "item_id"
	// RocketchatRoles - roles to fetch affiliation data for rocketchat messages
	RocketchatRoles = []string{"u"}
	// MustWaitRE - parse too many requests error message
	MustWaitRE = regexp.MustCompile(`must wait (\d+) seconds before`)
)

View Source

var (
	// StubRawMapping - Stub raw index mapping
	StubRawMapping = []byte(``)
	// StubRichMapping - Stub rich index mapping
	StubRichMapping = []byte(``)
	// StubCategories - categories defined for Stub
	StubCategories = map[string]struct{}{Stub: {}}
)

View Source

var (
	// EmailRegex - regexp to match email address
	EmailRegex = regexp.MustCompile("^[][a-zA-Z0-9.!#$%&'*+\\/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$")
	// EmailReplacer - replacer for some email buggy characters
	EmailReplacer = strings.NewReplacer(" at ", "@", " AT ", "@", " At ", "@", " dot ", ".", " DOT ", ".", " Dot ", ".", "<", "", ">", "", "`", "")

	// OpenAddrRE - '<...' -> '<' (... = whitespace)
	OpenAddrRE = regexp.MustCompile(`<\s+`)
	// CloseAddrRE - '...>' -> '>' (... = whitespace)
	CloseAddrRE = regexp.MustCompile(`\s+>`)
	// WhiteSpace - one or more whitespace characters
	WhiteSpace = regexp.MustCompile(`\s+`)
)

View Source

var (
	// LowerDayNames - downcased 3 letter US day names
	LowerDayNames = map[string]struct{}{
		"mon": {},
		"tue": {},
		"wed": {},
		"thu": {},
		"fri": {},
		"sat": {},
		"sun": {},
	}
	// LowerMonthNames - map lower month names
	LowerMonthNames = map[string]string{
		"jan": "Jan",
		"feb": "Feb",
		"mar": "Mar",
		"apr": "Apr",
		"may": "May",
		"jun": "Jun",
		"jul": "Jul",
		"aug": "Aug",
		"sep": "Sep",
		"oct": "Oct",
		"nov": "Nov",
		"dec": "Dec",
	}
	// LowerFullMonthNames - map lower month names (full)
	LowerFullMonthNames = map[string]string{
		"january":   "Jan",
		"february":  "Feb",
		"march":     "Mar",
		"april":     "Apr",
		"may":       "May",
		"june":      "Jun",
		"july":      "Jul",
		"august":    "Aug",
		"september": "Sep",
		"october":   "Oct",
		"november":  "Nov",
		"decdember": "Dec",
	}
	// SpacesRE - match 1 or more space characters
	SpacesRE = regexp.MustCompile(`\s+`)
	// TZOffsetRE - time zone offset that comes after +0... +1... -0... -1...
	// Can be 3 disgits or 3 digits then whitespace and then anything
	TZOffsetRE = regexp.MustCompile(`^(\d{3})(\s+.*$|$)`)
)

View Source

var (
	// GRedactedStrings - need to be global, to redact them from error logs
	GRedactedStrings map[string]struct{}
	// GRedactedMtx - guard access to this map while in MT
	GRedactedMtx *sync.RWMutex

	// AnonymizeURLPattern - used to remove sensitive data from the url - 3rd can be a GitHub password
	AnonymizeURLPattern = regexp.MustCompile(`(^.*)(://)(.*@)(.*$)`)
)

View Source

var (
	// MT - are we running in multiple threading mode?
	MT = false
)

Functions ¶

func AffsDataForRoles ¶

func AffsDataForRoles(ctx *Ctx, ds DS, rich map[string]interface{}, roles []string) (data map[string]interface{}, e error)

AffsDataForRoles - return affs data for given roles

func AffsIdentityIDs ¶

func AffsIdentityIDs(ctx *Ctx, ds DS, identity map[string]interface{}) (ids [2]interface{})

AffsIdentityIDs - returns affiliations identity id, uuid data uses internal cache

func AnonymizeURL ¶

func AnonymizeURL(url string) string

AnonymizeURL - remove sensitive data from the URL

func Base64DecodeCookies ¶

func Base64DecodeCookies(enc []byte) (cookies []string, err error)

Base64DecodeCookies - decode cookies stored as stream of bytes to array of strings

func Base64DecodeHeaders ¶

func Base64DecodeHeaders(enc []byte) (headers map[string][]string, err error)

Base64DecodeHeaders - decode headers stored as stream of bytes to map of string arrays

func Base64EncodeCookies ¶

func Base64EncodeCookies(cookies []string) (enc []byte)

Base64EncodeCookies - encode cookies array (strings) to base64 stream of bytes

func Base64EncodeHeaders ¶

func Base64EncodeHeaders(headers map[string][]string) (enc []byte)

Base64EncodeHeaders - encode headers to base64 stream of bytes

func BytesToStringTrunc ¶

func BytesToStringTrunc(data []byte, maxLen int, addLenInfo bool) (str string)

BytesToStringTrunc - truncate bytes stream to no more than maxLen

func CacheSummary ¶

func CacheSummary(ctx *Ctx)

CacheSummary - display cache summary stats

func CommonFields ¶

func CommonFields(ds DS, date interface{}, category string) (fields map[string]interface{})

CommonFields - common rich item fields { "is_dsname_category": 1, "grimoire_creation_date": dt}

func ConfluenceEnrichItemsFunc ¶

func ConfluenceEnrichItemsFunc(ctx *Ctx, ds DS, thrN int, items []interface{}, docs *[]interface{}) (err error)

ConfluenceEnrichItemsFunc - iterate items and enrich them items is a current pack of input items docs is a pointer to where extracted identities will be stored

func ConnectAffiliationsDB ¶

func ConnectAffiliationsDB(ctx *Ctx)

ConnectAffiliationsDB - connect to affilaitions DB

func CookieToString ¶

func CookieToString(c *http.Cookie) (s string)

CookieToString - convert cookie to string

func CopyAffsRoleData ¶

func CopyAffsRoleData(dst, src map[string]interface{}, dstRole, srcRole string)

CopyAffsRoleData - copy affiliations fields from source role to dest role

func CreateESCache ¶

func CreateESCache(ctx *Ctx)

CreateESCache - creates dads_cache index needed for caching

func DBUploadIdentitiesFunc ¶

func DBUploadIdentitiesFunc(ctx *Ctx, ds DS, thrN int, docs, outDocs *[]interface{}, last bool) (e error)

DBUploadIdentitiesFunc - function to upload identities to affiliation DB We assume here that docs maintained my iterator func contains a list of [3]string Each identity is [3]string [name, username, email] outDocs is maintained with DB bulk size last flag signalling that this is the last (so it must flush output then)

there can be no items in input pack in the last flush call

func DeepSet ¶

func DeepSet(m interface{}, ks []string, v interface{}, create bool) (err error)

DeepSet - set deep property of non-type decoded interface

func Dig ¶

func Dig(iface interface{}, keys []string, fatal, silent bool) (v interface{}, ok bool)

Dig interface for array of keys

func DumpKeys ¶

func DumpKeys(i interface{}) string

DumpKeys - dump interface structure, but only keys, no values

func DumpPreview ¶

func DumpPreview(i interface{}, l int) string

DumpPreview - dump interface structure, keys and truncated values preview

func ESBulkUploadFunc ¶

func ESBulkUploadFunc(ctx *Ctx, ds DS, thrN int, docs, outDocs *[]interface{}, last bool) (e error)

ESBulkUploadFunc - function to bulk upload items to ES We assume here that docs maintained my iterator func contains a list of rich items outDocs is maintained with ES bulk size last flag signalling that this is the last (so it must flush output then)

there can be no items in input pack in the last flush call

func ESCacheDelete ¶

func ESCacheDelete(ctx *Ctx, key string)

ESCacheDelete - delete cache key

func ESCacheDeleteExpired ¶

func ESCacheDeleteExpired(ctx *Ctx)

ESCacheDeleteExpired - delete expired cache entries

func ESCacheSet ¶

func ESCacheSet(ctx *Ctx, key string, entry *ESCacheEntry)

ESCacheSet - set cache value

func EmptyAffsItem ¶

func EmptyAffsItem(role string, undef bool) map[string]interface{}

EmptyAffsItem - return empty affiliations item for a given role

func Enrich ¶

func Enrich(ctx *Ctx, ds DS) (err error)

Enrich - implement fetch raw data (generic)

func EnrichComments ¶

func EnrichComments(ctx *Ctx, ds DS, comments []interface{}, item map[string]interface{}, affs bool) (richComments []interface{}, err error)

EnrichComments - return rich item from raw item for a given author type

func EnrichItem ¶

func EnrichItem(ctx *Ctx, ds DS, richItem map[string]interface{}) (err error)

EnrichItem - perform generic additional operations on already enriched item

func EnrichPairProgrammingItem ¶

func EnrichPairProgrammingItem(richItem map[string]interface{}) (err error)

EnrichPairProgrammingItem - additional operations on already enriched item for pair programming

func EnsurePath ¶

func EnsurePath(path string, noLastDir bool) (string, error)

EnsurePath - craete archive directory (and all necessary parents as well) if noLastDir is set, then skip creating the last directory in the path

func ExecCommand ¶

func ExecCommand(ctx *Ctx, cmdAndArgs []string, cwd string, env map[string]string) (sout, serr string, err error)

ExecCommand - execute command given by array of strings with eventual environment map

func ExecCommandPipe ¶

func ExecCommandPipe(ctx *Ctx, cmdAndArgs []string, cwd string, env map[string]string) (stdOutPipe io.ReadCloser, cmd *exec.Cmd, err error)

ExecCommandPipe - execute command given by array of strings with eventual environment map, return STDOUT pipe to read from

func ExecDB ¶

func ExecDB(ctx *Ctx, query string, args ...interface{}) (res sql.Result, err error)

ExecDB - execute DB query without transaction

func ExecSQL ¶

func ExecSQL(ctx *Ctx, tx *sql.Tx, query string, args ...interface{}) (sql.Result, error)

ExecSQL - execute db query with transaction if provided

func ExecTX ¶

func ExecTX(ctx *Ctx, tx *sql.Tx, query string, args ...interface{}) (res sql.Result, err error)

ExecTX - execute DB query with transaction

func ExecuteAffiliationsAPICall ¶

func ExecuteAffiliationsAPICall(ctx *Ctx, method, path string, cacheToken bool) (data map[string]interface{}, err error)

ExecuteAffiliationsAPICall - execute a call to Affiliations API

func FatalOnError ¶

func FatalOnError(err error) string

FatalOnError displays error message (if error present) and exits program

func Fatalf ¶

func Fatalf(f string, a ...interface{})

Fatalf - it will call FatalOnError using fmt.Errorf with args provided

func FetchRaw ¶

func FetchRaw(ctx *Ctx, ds DS) (err error)

FetchRaw - implement fetch raw data (generic)

func FilterRedacted ¶

func FilterRedacted(str string) string

FilterRedacted - filter out all known redacted starings

func FindObject ¶

func FindObject(ctx *Ctx, object, key, id string, fields []string) (obj map[string]interface{}, err error)

FindObject - fetch given fields from object (identities, profiles, uidentities etc.) having key=id Assuming that given object has an unique key to gte it

func ForEachESItem ¶

func ForEachESItem(
	ctx *Ctx,
	ds DS,
	raw bool,
	ufunct func(*Ctx, DS, int, *[]interface{}, *[]interface{}, bool) error,
	uitems func(*Ctx, DS, int, []interface{}, *[]interface{}) error,
	cacheFor *time.Duration,
	mt bool,
) (err error)

ForEachESItem - perform specific function for all raw/rich items ufunct: function to perform on input pack, receives input pack, pointer to an output pack

and a flag signalling that this is the last (so it must flush output then)
there can be no items in input pack in the last flush call

uitems: function to extract items from input data: can just add documents, but can also maintain a pack of documents identities

receives items and pointer to output items (which then become input for ufunct)

func GerritEnrichItemsFunc ¶

func GerritEnrichItemsFunc(ctx *Ctx, ds DS, thrN int, items []interface{}, docs *[]interface{}) (err error)

GerritEnrichItemsFunc - iterate items and enrich them items is a current pack of input items docs is a pointer to where extracted identities will be stored

func GetAPIToken ¶

func GetAPIToken() (string, error)

GetAPIToken - return an API token to use dev-analytics-api API calls If JWT_TOKEN env is specified - just use the provided token without any checks Else get auth0 data from AUTH0_DATA and generate/reuse a token stored in ES cache

func GetESCache ¶

func GetESCache(ctx *Ctx, k string) (b []byte, tg string, expires time.Time, ok bool)

GetESCache - get value from cache - thread safe and support expiration

func GetEnrollments ¶

func GetEnrollments(ctx *Ctx, ds DS, uuid string, dt time.Time, single bool) (orgs []string, e error)

GetEnrollments - returns enrollments for a given uuid in a given date, possibly multiple uses cache with date resolution (pslug,uuid,dt.YYYYMMDD)

func GetEnrollmentsBoth ¶

func GetEnrollmentsBoth(ctx *Ctx, ds DS, uuid string, dt time.Time) (org string, orgs []string, e error)

GetEnrollmentsBoth - returns org name(s) for given uuid and name returns data returned by bot GetEnrollmentsSingle and GetEnrollmentsMulti by using a single HTTP request when both were not yet called for a given key

func GetEnrollmentsMulti ¶

func GetEnrollmentsMulti(ctx *Ctx, ds DS, uuid string, dt time.Time) (orgs []string, e error)

GetEnrollmentsMulti - returns org name(s) for given uuid and name Returns 1 or more organizations (all that matches the current date) If none matches it returns array Unknown

func GetEnrollmentsSingle ¶

func GetEnrollmentsSingle(ctx *Ctx, ds DS, uuid string, dt time.Time) (org string, e error)

GetEnrollmentsSingle - returns org name (or Unknown) for given uuid and date

func GetIdentityUUID ¶

func GetIdentityUUID(ctx *Ctx, ds DS, id string) (uuid interface{})

GetIdentityUUID - identity's UUID for a given ID uses internal cache

func GetL2Cache ¶

func GetL2Cache(ctx *Ctx, k string) (b []byte, ok bool)

GetL2Cache - get value from cache - thread safe and support expiration

func GetLastOffset ¶

func GetLastOffset(ctx *Ctx, ds DS, raw bool) (offset float64)

GetLastOffset - get last offset from ElasticSearch

func GetLastUpdate ¶

func GetLastUpdate(ctx *Ctx, ds DS, raw bool) (lastUpdate *time.Time)

GetLastUpdate - get last update date from ElasticSearch

func GetRedacted ¶

func GetRedacted() (str string)

GetRedacted - get redacted

func GetThreadsNum ¶

func GetThreadsNum(ctx *Ctx) int

GetThreadsNum returns the number of available CPUs If environment variable DA_DS_ST is set it retuns 1 It can be used to debug single threaded verion

func GitEnrichItemsFunc ¶

func GitEnrichItemsFunc(ctx *Ctx, ds DS, thrN int, items []interface{}, docs *[]interface{}) (err error)

GitEnrichItemsFunc - iterate items and enrich them items is a current pack of input items docs is a pointer to where extracted identities will be stored

func GitHubEnrichItemsFunc ¶

func GitHubEnrichItemsFunc(ctx *Ctx, ds DS, thrN int, items []interface{}, docs *[]interface{}) (err error)

GitHubEnrichItemsFunc - iterate items and enrich them items is a current pack of input items docs is a pointer to where extracted identities will be stored

func GroupsioEnrichItemsFunc ¶

func GroupsioEnrichItemsFunc(ctx *Ctx, ds DS, thrN int, items []interface{}, docs *[]interface{}) (err error)

GroupsioEnrichItemsFunc - iterate items and enrich them items is a current pack of input items docs is a pointer to where extracted identities will be stored

func HandleMapping ¶

func HandleMapping(ctx *Ctx, ds DS, raw bool) (err error)

HandleMapping - create/update mapping for raw or rich index

func IdentityAffsData ¶

func IdentityAffsData(ctx *Ctx, ds DS, identity map[string]interface{}, aid interface{}, dt time.Time, role string) (outItem map[string]interface{}, empty bool, e error)

IdentityAffsData - add affiliations related data identity - full identity aid identity ID value (which is uuid), for example from "author_id", "creator_id" etc. either identity or aid must be specified

func IdentityAffsDomain ¶

func IdentityAffsDomain(identity map[string]interface{}) (domain interface{}, email string)

IdentityAffsDomain -return domain for given identity using email if specified

func IndexAt ¶

func IndexAt(s, sep string, n int) int

IndexAt - index of substring starting at a given position

func InitializeAuth0 ¶

func InitializeAuth0() error

InitializeAuth0 - initializes Auth0 client using data stored in AUTH0_DATA

func InterfaceToStringTrunc ¶

func InterfaceToStringTrunc(iface interface{}, maxLen int, addLenInfo bool) (str string)

InterfaceToStringTrunc - truncate interface representation

func IsOldFormat ¶

func IsOldFormat(ctx *Ctx, idx string) (old bool)

IsOldFormat - is this an old format index (bitergia one)?

func IsValidDomain ¶

func IsValidDomain(domain string) (valid bool)

IsValidDomain - is MX domain valid? uses internal cache

func IsValidEmail ¶

func IsValidEmail(email string, validateDomain, guess bool) (valid bool, newEmail string)

IsValidEmail - is email correct: len, regexp, MX domain uses internal cache

func ItemsIdentitiesFunc ¶

func ItemsIdentitiesFunc(ctx *Ctx, ds DS, thrN int, items []interface{}, docs *[]interface{}) (err error)

ItemsIdentitiesFunc - extract identities from items items is a current pack of ES input items docs is a pointer to where extracted identities will be stored each identity is [3]string [name, username, email]

func ItemsRefreshIdentitiesFunc ¶

func ItemsRefreshIdentitiesFunc(ctx *Ctx, ds DS, thrN int, richItems []interface{}, docs *[]interface{}) (err error)

ItemsRefreshIdentitiesFunc - refresh input items/re-enrich items is a current pack of ES rich items docs is a pointer to where updated rich items will be stored

func JSONEscape ¶

func JSONEscape(str string) string

JSONEscape - escape string for JSON to avoid injections

func JiraEnrichItemsFunc ¶

func JiraEnrichItemsFunc(ctx *Ctx, ds DS, thrN int, items []interface{}, docs *[]interface{}) (err error)

JiraEnrichItemsFunc - iterate items and enrich them items is a current pack of input items docs is a pointer to where extracted identities will be stored

func KeysOnly ¶

func KeysOnly(i interface{}) (o map[string]interface{})

KeysOnly - return a corresponding interface contining only keys

func MatchGroups ¶

func MatchGroups(re *regexp.Regexp, arg string) (result map[string]string)

MatchGroups - return regular expression matching groups as a map

func MatchGroupsArray ¶

func MatchGroupsArray(re *regexp.Regexp, arg string) (result map[string][]string)

MatchGroupsArray - return regular expression matching groups as a map

func MaybeESCacheCleanup ¶

func MaybeESCacheCleanup(ctx *Ctx)

MaybeESCacheCleanup - chance of cleaning expired cache entries

func MaybeMemCacheCleanup ¶

func MaybeMemCacheCleanup(ctx *Ctx)

MaybeMemCacheCleanup - chance of cleaning expired cache entries

func MemCacheDeleteExpired ¶

func MemCacheDeleteExpired(ctx *Ctx)

MemCacheDeleteExpired - delete expired cache entries

func NoSSLVerify ¶

func NoSSLVerify()

NoSSLVerify - turn off SSL validation

func ParseAddresses ¶

func ParseAddresses(ctx *Ctx, addrs string, maxAddrs int) (emails []*mail.Address, ok bool)

ParseAddresses - parse address string into one or more name/email pairs

func ParseDateWithTz ¶

func ParseDateWithTz(indt string) (dt, dtInTz time.Time, off float64, valid bool)

ParseDateWithTz - try to parse mbox date

func ParseMBoxMsg ¶

func ParseMBoxMsg(ctx *Ctx, groupName string, msg []byte) (item map[string]interface{}, valid, warn bool)

ParseMBoxMsg - parse a raw MBox message into object to be inserte dinto raw ES

func PartitionString ¶

func PartitionString(s string, sep string) [3]string

PartitionString - partition a string to [pre-sep, sep, post-sep]

func PeriodParse ¶

func PeriodParse(perStr string) (dur time.Duration, ok bool)

PeriodParse - tries to parse period

func PostprocessFields ¶

func PostprocessFields(outItem map[string]interface{}, role, email string)

PostprocessFields - check name field, if it is empty then copy from email (if not empty) or username (if not empty) Then check name and username - it cannot contain email addess, if it does - replace a@domain with a-MISSING-NAME

func PostprocessNameUsername ¶

func PostprocessNameUsername(name, username, email string) (outName, outUsername string)

PostprocessNameUsername - check name field, if it is empty then copy from email (if not empty) or username (if not empty) Then check name and username - it cannot contain email addess, if it does - replace a@domain with a-MISSING-NAME

func PrettyPrintJSON ¶

func PrettyPrintJSON(jsonBytes []byte) []byte

PrettyPrintJSON - pretty formats raw JSON bytes

func PreviewOnly ¶

func PreviewOnly(i interface{}, l int) (o interface{})

PreviewOnly - return a corresponding interface with preview values

func PrintCache ¶

func PrintCache(iCache interface{}) (s string)

PrintCache - pretty print cache entries

func Printf ¶

func Printf(format string, args ...interface{})

Printf is a wrapper around Printf(...) that supports logging and removes redacted data.

func PrintfNoRedacted ¶

func PrintfNoRedacted(format string, args ...interface{})

PrintfNoRedacted is a wrapper around Printf(...) that supports logging and don't removes redacted data

func ProgressInfo ¶

func ProgressInfo(i, n int, start time.Time, last *time.Time, period time.Duration, msg string)

ProgressInfo display info about progress: i/n if current time >= last + period If displayed info, update last

func QueryDB ¶

func QueryDB(ctx *Ctx, query string, args ...interface{}) (rows *sql.Rows, err error)

QueryDB - query database without transaction

func QueryOut ¶

func QueryOut(ctx *Ctx, in bool, err error, query string, args ...interface{})

QueryOut - display DB query

func QuerySQL ¶

func QuerySQL(ctx *Ctx, tx *sql.Tx, query string, args ...interface{}) (*sql.Rows, error)

QuerySQL - query DB using transaction if provided

func QueryTX ¶

func QueryTX(ctx *Ctx, tx *sql.Tx, query string, args ...interface{}) (rows *sql.Rows, err error)

QueryTX - query database with transaction

func QueryToStringArray ¶

func QueryToStringArray(ctx *Ctx, query string, args ...interface{}) (res []string)

QueryToStringArray - execute SQL query returning multiple rows each containitg a single string column

func QueryToStringIntArrays ¶

func QueryToStringIntArrays(ctx *Ctx, query string, args ...interface{}) (sa []string, ia []int64)

QueryToStringIntArrays - execute SQL query returning multiple rows each containitg (string,int64)

func RedactEmail ¶

func RedactEmail(in, suff string, forceSuff bool) string

RedactEmail - possibly redact email from "in" If in contains @, replace part after last "@" with suff If in doesn't contain "@" then return it or (if forceSuff is set) return in + suff

func RefreshIdentities ¶

func RefreshIdentities(ctx *Ctx, ds DS) (err error)

RefreshIdentities - refresh identities We iterate over rich index to refresh its affiliation data

func Request ¶

func Request(
	ctx *Ctx,
	url, method string,
	headers map[string]string,
	payload []byte,
	cookies []string,
	jsonStatuses, errorStatuses, okStatuses, cacheStatuses map[[2]int]struct{},
	retryRequest bool,
	cacheFor *time.Duration,
	skipInDryRun bool,
) (result interface{}, status int, outCookies []string, outHeaders map[string][]string, err error)

Request - wrapper around RequestNoRetry supporting retries

func RequestNoRetry ¶

func RequestNoRetry(
	ctx *Ctx,
	url, method string,
	headers map[string]string,
	payload []byte,
	cookies []string,
	jsonStatuses, errorStatuses, okStatuses, cacheStatuses map[[2]int]struct{},
) (result interface{}, status int, isJSON bool, outCookies []string, outHeaders map[string][]string, cache bool, err error)

RequestNoRetry - wrapper to do any HTTP request jsonStatuses - set of status code ranges to be parsed as JSONs errorStatuses - specify status value ranges for which we should return error okStatuses - specify status value ranges for which we should return error (only taken into account if not empty)

func ResetThreadsNum ¶

func ResetThreadsNum(ctx *Ctx)

ResetThreadsNum - allows clearing current setting so the new one can be applied

func ResetUUIDCache ¶

func ResetUUIDCache()

ResetUUIDCache - resets cache

func RocketchatEnrichItemsFunc ¶

func RocketchatEnrichItemsFunc(ctx *Ctx, ds DS, thrN int, items []interface{}, docs *[]interface{}) (err error)

RocketchatEnrichItemsFunc - iterate items and enrich them items is a current pack of input items docs is a pointer to where extracted identities will be stored

func SendMultipleDocumentsToGAP ¶

func SendMultipleDocumentsToGAP(ctx *Ctx, indexNames, docIDs []string, docs [][]byte, cacheToken bool) (err error)

SendMultipleDocumentsToGAP - send failed ES bulk to GAP API

func SendSingleDocumentToGAP ¶

func SendSingleDocumentToGAP(ctx *Ctx, indexName, docID string, doc []byte, cacheToken bool) (err error)

SendSingleDocumentToGAP - send failed ES item to GAP API

func SendToElastic ¶

func SendToElastic(ctx *Ctx, ds DS, raw bool, key string, items []interface{}) (err error)

SendToElastic - send items to ElasticSearch

func SetDBSessionOrigin ¶

func SetDBSessionOrigin(ctx *Ctx) (err error)

SetDBSessionOrigin - Set Session DB variable @origin to 'dads' so we will know which tool performed the DB operation

func SetESCache ¶

func SetESCache(ctx *Ctx, k, tg string, b []byte, expires time.Duration)

SetESCache - set cache value, expiration date and handles multithreading etc

func SetL2Cache ¶

func SetL2Cache(ctx *Ctx, k, tg string, b []byte, expires time.Duration)

SetL2Cache - set cache value, expiration date and handles multithreading etc

func SetMT ¶

func SetMT()

SetMT - we're in multithreaded mode, setup global caches mutexes

func SleepForRateLimit ¶

func SleepForRateLimit(ctx *Ctx, ds DS, rateLimit, rateLimitReset, minRate int, waitRate bool) (err error)

SleepForRateLimit - sleep for rate or return error when rate exceeded

func StandardItemsFunc ¶

func StandardItemsFunc(ctx *Ctx, ds DS, items []interface{}, docs *[]interface{}) (err error)

StandardItemsFunc - just get each doument's _source and append to output docs items is a current pack of input items docs is a pointer to where extracted items will be stored

func StringToBool ¶

func StringToBool(v string) bool

StringToBool - convert string value to boolean value returns false for anything that was parsed as false, zero, empty etc: f, F, false, False, fALSe, 0, "", 0.00 else returns true

func StringToCookie ¶

func StringToCookie(s string) (c *http.Cookie)

StringToCookie - convert string to cookie

func StringTrunc ¶

func StringTrunc(data string, maxLen int, addLenInfo bool) (str string)

StringTrunc - truncate string to no more than maxLen

func StubEnrichItemsFunc ¶

func StubEnrichItemsFunc(ctx *Ctx, ds DS, thrN int, items []interface{}, docs *[]interface{}) (err error)

StubEnrichItemsFunc - iterate items and enrich them items is a current pack of input items docs is a pointer to where extracted identities will be stored

func TimeParseAny ¶

func TimeParseAny(dtStr string) (time.Time, error)

TimeParseAny - attempts to parse time from string YYYY-MM-DD HH:MI:SS Skipping parts from right until only YYYY id left

func TimeParseES ¶

func TimeParseES(dtStr string) (time.Time, error)

TimeParseES - parse datetime in ElasticSearch output format

func TimeParseESSec ¶

func TimeParseESSec(dtStr string) (time.Time, error)

TimeParseESSec - parse datetime in ElasticSearch output format

func TimeParseInterfaceString ¶

func TimeParseInterfaceString(date interface{}) (dt time.Time, err error)

TimeParseInterfaceString - parse interface{} -> string -> time.Time

func ToESDate ¶

func ToESDate(dt time.Time) string

ToESDate - return time formatted as YYYY-MM-DDTHH:MI:SS.uuuuuu+00:00

func ToYMDDate ¶

func ToYMDDate(dt time.Time) string

ToYMDDate - return time formatted as YYYYMMDD

func ToYMDHMDate ¶

func ToYMDHMDate(dt time.Time) string

ToYMDHMDate - return time formatted as YYYY-MM-DD HH:MI

func ToYMDHMSDate ¶

func ToYMDHMSDate(dt time.Time) string

ToYMDHMSDate - return time formatted as YYYY-MM-DD HH:MI:SS

func ToYMDTHMSZDate ¶

func ToYMDTHMSZDate(dt time.Time) string

ToYMDTHMSZDate - return time formatted as YYYY-MM-DDTHH:MI:SSZ

func UUIDAffs ¶

func UUIDAffs(ctx *Ctx, args ...string) (h string)

UUIDAffs - generate UUID of string args uses internal cache downcases arguments, all but first can be empty

func UUIDNonEmpty ¶

func UUIDNonEmpty(ctx *Ctx, args ...string) (h string)

UUIDNonEmpty - generate UUID of string args (all must be non-empty) uses internal cache used to generate document UUID's

func UpdateRateLimit ¶

func UpdateRateLimit(ctx *Ctx, ds DS, headers map[string][]string, rateLimitHeader, rateLimitResetHeader string) (rateLimit, rateLimitReset, secondsToReset int)

UpdateRateLimit - generic function to get rate limit data from header

func UploadIdentities ¶

func UploadIdentities(ctx *Ctx, ds DS) (err error)

UploadIdentities - upload identities to SH DB We assume here that docs maintained my iterator func contains a list of [3]string Each identity is [3]string [name, username, email]

Types ¶

type BugZilla ¶

type BugZilla struct {
	Origin      *Flag
	EsIndex     *Flag
	FromDate    *Flag
	Project     *Flag
	DoFetch     *Flag
	DoEnrich    *Flag
	FetchSize   *Flag
	EnrichSize  *Flag
	ProjectSlug *Flag
}

BugZilla parameter context contains all required parameters to run Bugzilla fetch and enrich

type Ctx ¶

type Ctx struct {
	DS                 string     // From DA_DS: ds type: for example jira, gerrit, slack etc., other env variablse will use this as a prefix
	DSPrefix           string     // uppercase(DS) + _: if DS is "slack" then prefix would be "DA_SLACK_"
	Debug              int        // From DA_DS_DEBUG Debug level: 0-no, 1-info, 2-verbose
	DebugSQL           int        // From DA_DS_DEBUG_SQL SQL Debug level
	Retry              int        // From DA_DS_RETRY: how many times retry failed operatins, default 5
	ST                 bool       // From DA_DS_ST true: use single threaded version, false: use multi threaded version, default false
	NCPUs              int        // From DA_DS_NCPUS, set to override number of CPUs to run, this overwrites DA_DS_ST, default 0 (which means do not use it, use all CPU reported by go library)
	NCPUsScale         float64    // From DA_DS_NCPUS_SCALE, scale number of CPUs, for example 2.0 will report number of cpus 2.0 the number of actually available CPUs
	Enrich             bool       // From DA_DS_ENRICH, flag to run enrichment
	RawIndex           string     // From DA_DS_RAW_INDEX - raw index name
	RichIndex          string     // From DA_DS_RICH_INDEX - rich index name
	Tag                string     // From DA_DS_TAG - tag
	ESURL              string     // From DA_DS_ES_URL - ElasticSearch URL
	AffiliationAPIURL  string     // From DA_DS_AFFILIATION_API_URL - Affiliation API URL
	ESBulkSize         int        // From DA_DS_ES_BULK_SIZE - ElasticSearch bulk size
	ESScrollSize       int        // From DA_DS_ES_SCROLL_SIZE - ElasticSearch scroll size
	ESScrollWait       string     // From DA_DS_ES_SCROLL_WAIT - ElasticSearch scroll wait
	DBBulkSize         int        // From DA_DS_DB_BULK_SIZE - affiliations DB bulk size
	DBHost             string     // From DA_DS_DB_HOST - affiliation DB host
	DBName             string     // From DA_DS_DB_NAME - affiliation DB name
	DBUser             string     // From DA_DS_DB_USER - affiliation DB user
	DBPass             string     // From DA_DS_DB_PASS - affiliation DB pass
	DBPort             string     // From DA_DS_DB_PORT - affiliation DB port
	DBOpts             string     // From DA_DS_DB_OPTS - affiliation DB & separated iURL encoded options, for example "charset=utf8&parseTime=true"
	DBConn             string     // From DA_DS_DB_CONN - affiliation DB conn (full connection string - if set no other DB params will be used)
	NoRaw              bool       // From DA_DS_NO_RAW - do only the enrichment
	NoIdentities       bool       // From DA_DS_NO_IDENTITIES - do not upload identities to affiliations database (if you want to perform enrichment - only use this when you did that at least once)
	NoCache            bool       // From DA_DS_NO_CACHE - do not use L2(mem, ES) cache for selected requests
	NoAffiliation      bool       // From DA_DS_NO_AFFILIATION, if set, no enrollemnts affiliations will be processed
	DryRun             bool       // From DA_DS_DRY_RUN - do only requests that read data, no write to anything (excluding cache - this one can be written in dry-run mode - still can be disabled with NoCache)
	RefreshAffs        bool       // From DA_DS_REFRESH_AFFS - refresh affiliation data
	OnlyIdentities     bool       // From DA_DS_ONLY_IDENTITIES - only add identities to affiliation database
	ForceFull          bool       // From DA_DS_FORCE_FULL - force running full data source enrichment, do not attempt to detect where to start from
	Project            string     // From DA_DS_PROJECT - set project can be for example "ONAP"
	ProjectSlug        string     // From DA_DS_PROJECT_SLUG - set project slug - fixture slug, for example "lfn/onap"
	ProjectFilter      bool       // From DA_DS_PROJECT_FILTER - set project filter (if this is set for example via 'p2o: true' fixture flag, this says that DS should internally filter by project, otherwise it means that no internal project filtering is needed and this is only used to set project on ES documents)
	Category           string     // From DA_DS_CATEGORY - set category (some DS support this), for example "issue" (github/issue, github/pull_request etc.)
	Groups             []string   // From GROUPS, always contain ProjectSlug at the minimum
	DateFrom           *time.Time // From DA_DS_DATE_FROM
	DateTo             *time.Time // From DA_DS_DATE_TO
	OffsetFrom         float64    // From DA_DS_OFFSET_FROM
	OffsetTo           float64    // From DA_DS_OFFSET_TO
	LegacyUUID         bool       // From DA_DS_LEGACY_UUID - use python code for generating uuids
	DropRich           bool       // From DA_DS_DROP_RICH - drop rich index (this is to allow regenerating data), index is only dropped when it is in old (bitergia) format, so we avoid dropping it for every origin
	DropRaw            bool       // From DA_DS_DROP_RAW - drop raw index (this is to allow regenerating data), index is only dropped when it is in old (bitergia) format, so we avoid dropping it for every origin
	AllowFail          int        // From DA_DS_ALLOW_FAIL - allow fail uploading single documents to elastic: 0 - send to GAP handler and continue, 1 - don't allow, 2-allow fail, if failed, skip entire pack (ignore), 3-allow fail, but each next document without retries, else-allow fail and retry each individual document
	CheckAuthorID      bool       // From DA_DS_CHECK_AID - so additional check if a rich document has author_id field, skip the document if it doesn't
	AffsAPIFailFatal   bool       // From DA_AFFS_API_FAIL_FATAL (note no DA_DS_ prefix - this is a global flag starting with DA_)
	DateFromDetected   bool
	OffsetFromDetected bool
	DB                 *sqlx.DB
	ESScrollWaitSecs   float64
	GapURL             string
	Retries            uint
	Delay              time.Duration
	Repository         []Repository
	AffAPI             string
	SlackWebHookURL    string
	// Bugzilla contains all bugzilla params
	BugZilla *BugZilla

	PiperMail *PiperMail

	GoogleGroups *GoogleGroups
}

Ctx - environment context packed in structure

func (*Ctx) AffsDBConfigured ¶

func (ctx *Ctx) AffsDBConfigured() bool

AffsDBConfigured - is affiliations DB configured?

func (*Ctx) BoolEnv ¶

func (ctx *Ctx) BoolEnv(k string) bool

BoolEnv - parses env variable as bool returns false for anything that was parsed as false, zero, empty etc: f, F, false, False, fALSe, 0, "", 0.00 else returns true

func (*Ctx) Env ¶

func (ctx *Ctx) Env(v string) string

Env - get env value using current DS prefix

func (Ctx) Info ¶

func (ctx Ctx) Info() string

Info - return context in human readable form

func (*Ctx) Init ¶

func (ctx *Ctx) Init()

Init - get context from environment variables

func (*Ctx) ParseFlags ¶

func (ctx *Ctx) ParseFlags()

ParseFlags declare and parse CLI flags

func (*Ctx) Print ¶

func (ctx *Ctx) Print()

Print context contents

func (*Ctx) Validate ¶

func (ctx *Ctx) Validate() (err error)

Validate - check if config is correct

type DS ¶

type DS interface {
	ParseArgs(*Ctx) error
	Name() string
	Info() string
	Validate(*Ctx) error
	FetchRaw(*Ctx) error
	FetchItems(*Ctx) error
	Enrich(*Ctx) error
	DateField(*Ctx) string
	OffsetField(*Ctx) string
	OriginField(*Ctx) string
	Categories() map[string]struct{}
	CustomFetchRaw() bool
	CustomEnrich() bool
	SupportDateFrom() bool
	SupportOffsetFrom() bool
	ResumeNeedsOrigin(*Ctx, bool) bool
	ResumeNeedsCategory(*Ctx, bool) bool
	Origin(*Ctx) string
	ItemID(interface{}) string
	RichIDField(*Ctx) string
	RichAuthorField(*Ctx) string
	ItemUpdatedOn(interface{}) time.Time
	ItemCategory(interface{}) string
	ElasticRawMapping() []byte
	ElasticRichMapping() []byte
	AddMetadata(*Ctx, interface{}) map[string]interface{}
	GetItemIdentities(*Ctx, interface{}) (map[[3]string]struct{}, error)
	EnrichItems(*Ctx) error
	EnrichItem(*Ctx, map[string]interface{}, string, bool, interface{}) (map[string]interface{}, error)
	AffsItems(*Ctx, map[string]interface{}, []string, interface{}) (map[string]interface{}, error)
	GetRoleIdentity(*Ctx, map[string]interface{}, string) map[string]interface{}
	AllRoles(*Ctx, map[string]interface{}) ([]string, bool)
	CalculateTimeToReset(*Ctx, int, int) int
	HasIdentities() bool
	UseDefaultMapping(*Ctx, bool) bool
}

DS - interface for all data source types

type DSConfluence ¶

type DSConfluence struct {
	DS          string
	URL         string // From DA_CONFLUENCE_URL - Group name like GROUP-topic
	NoSSLVerify bool   // From DA_CONFLUENCE_NO_SSL_VERIFY
	MultiOrigin bool   // From DA_CONFLUENCE_MULTI_ORIGIN - allow multiple groups in a single index
	MaxContents int    // From DA_CONFLUENCE_MAX_CONTENTS, defaults to ConfluenceDefaultMaxContents (200)
	User        string // From DA_CONFLUENCE_USER - if user is provided then we assume that we don't have base64 encoded user:token yet
	Token       string // From DA_CONFLUENCE_TOKEN - if user is not specified we assume that token already contains "<username>:<your-api-token>"
}

DSConfluence - DS implementation for confluence - does nothing at all, just presents a skeleton code

func (*DSConfluence) AddMetadata ¶

func (j *DSConfluence) AddMetadata(ctx *Ctx, item interface{}) (mItem map[string]interface{})

AddMetadata - add metadata to the item

func (*DSConfluence) AffsItems ¶

func (j *DSConfluence) AffsItems(ctx *Ctx, page map[string]interface{}, roles []string, date interface{}) (affsItems map[string]interface{}, err error)

AffsItems - return affiliations data items for given roles and date

func (*DSConfluence) AllRoles ¶

func (j *DSConfluence) AllRoles(ctx *Ctx, item map[string]interface{}) ([]string, bool)

AllRoles - return all roles defined for the backend roles can be static (always the same) or dynamic (per item) second return parameter is static mode (true/false) dynamic roles will use item to get its roles

func (*DSConfluence) CalculateTimeToReset ¶

func (j *DSConfluence) CalculateTimeToReset(ctx *Ctx, rateLimit, rateLimitReset int) (seconds int)

CalculateTimeToReset - calculate time to reset rate limits based on rate limit value and rate limit reset value

func (*DSConfluence) Categories ¶

func (j *DSConfluence) Categories() map[string]struct{}

Categories - return a set of configured categories

func (*DSConfluence) CustomEnrich ¶

func (j *DSConfluence) CustomEnrich() bool

CustomEnrich - is this datasource using custom enrich implementation?

func (*DSConfluence) CustomFetchRaw ¶

func (j *DSConfluence) CustomFetchRaw() bool

CustomFetchRaw - is this datasource using custom fetch raw implementation?

func (*DSConfluence) DateField ¶

func (j *DSConfluence) DateField(*Ctx) string

DateField - return date field used to detect where to restart from

func (*DSConfluence) ElasticRawMapping ¶

func (j *DSConfluence) ElasticRawMapping() []byte

ElasticRawMapping - Raw index mapping definition

func (*DSConfluence) ElasticRichMapping ¶

func (j *DSConfluence) ElasticRichMapping() []byte

ElasticRichMapping - Rich index mapping definition

func (*DSConfluence) Enrich ¶

func (j *DSConfluence) Enrich(ctx *Ctx) (err error)

Enrich - implement enrich data for datasource

func (*DSConfluence) EnrichItem ¶

func (j *DSConfluence) EnrichItem(ctx *Ctx, item map[string]interface{}, author string, affs bool, extra interface{}) (rich map[string]interface{}, err error)

EnrichItem - return rich item from raw item for a given author type

func (*DSConfluence) EnrichItems ¶

func (j *DSConfluence) EnrichItems(ctx *Ctx) (err error)

EnrichItems - perform the enrichment

func (*DSConfluence) FetchItems ¶

func (j *DSConfluence) FetchItems(ctx *Ctx) (err error)

FetchItems - implement enrich data for confluence datasource

func (*DSConfluence) FetchRaw ¶

func (j *DSConfluence) FetchRaw(ctx *Ctx) (err error)

FetchRaw - implement fetch raw data for datasource

func (*DSConfluence) GetConfluenceContents ¶

func (j *DSConfluence) GetConfluenceContents(ctx *Ctx, fromDate, next string) (contents []map[string]interface{}, newNext string, err error)

GetConfluenceContents - get confluence historical contents

func (*DSConfluence) GetHistoricalContents ¶

func (j *DSConfluence) GetHistoricalContents(ctx *Ctx, content map[string]interface{}, dateFrom time.Time) (contents []map[string]interface{}, err error)

GetHistoricalContents - get historical contents from teh current content

func (*DSConfluence) GetItemIdentities ¶

func (j *DSConfluence) GetItemIdentities(ctx *Ctx, doc interface{}) (identities map[[3]string]struct{}, err error)

GetItemIdentities return list of item's identities, each one is [3]string (name, username, email) tripples, special value Nil "none" means null we use string and not *string which allows nil to allow usage as a map key

func (*DSConfluence) GetRoleIdentity ¶

func (j *DSConfluence) GetRoleIdentity(ctx *Ctx, item map[string]interface{}, role string) (identity map[string]interface{})

GetRoleIdentity - return identity data for a given role

func (*DSConfluence) HasIdentities ¶

func (j *DSConfluence) HasIdentities() bool

HasIdentities - does this data source support identity data

func (DSConfluence) Info ¶

func (j DSConfluence) Info() string

Info - return DS configuration in a human readable form

func (*DSConfluence) ItemCategory ¶

func (j *DSConfluence) ItemCategory(item interface{}) string

ItemCategory - return unique identifier for an item

func (*DSConfluence) ItemID ¶

func (j *DSConfluence) ItemID(item interface{}) string

ItemID - return unique identifier for an item

func (*DSConfluence) ItemUpdatedOn ¶

func (j *DSConfluence) ItemUpdatedOn(item interface{}) time.Time

ItemUpdatedOn - return updated on date for an item

func (*DSConfluence) Name ¶

func (j *DSConfluence) Name() string

Name - return data source name

func (*DSConfluence) OffsetField ¶

func (j *DSConfluence) OffsetField(*Ctx) string

OffsetField - return offset field used to detect where to restart from

func (*DSConfluence) Origin ¶

func (j *DSConfluence) Origin(ctx *Ctx) string

Origin - return current origin

func (*DSConfluence) OriginField ¶

func (j *DSConfluence) OriginField(ctx *Ctx) string

OriginField - return origin field used to detect where to restart from

func (*DSConfluence) ParseArgs ¶

func (j *DSConfluence) ParseArgs(ctx *Ctx) (err error)

ParseArgs - parse confluence specific environment variables

func (*DSConfluence) ResumeNeedsCategory ¶

func (j *DSConfluence) ResumeNeedsCategory(ctx *Ctx, raw bool) bool

ResumeNeedsCategory - is category field needed when resuming Category should be needed when multiple types of categories save to the same index or there are multiple types of documents within the same category

func (*DSConfluence) ResumeNeedsOrigin ¶

func (j *DSConfluence) ResumeNeedsOrigin(ctx *Ctx, raw bool) bool

ResumeNeedsOrigin - is origin field needed when resuming Origin should be needed when multiple configurations save to the same index

func (*DSConfluence) RichAuthorField ¶

func (j *DSConfluence) RichAuthorField(*Ctx) string

RichAuthorField - return rich author field name

func (*DSConfluence) RichIDField ¶

func (j *DSConfluence) RichIDField(*Ctx) string

RichIDField - return rich ID field name

func (*DSConfluence) SupportDateFrom ¶

func (j *DSConfluence) SupportDateFrom() bool

SupportDateFrom - does DS support resuming from date?

func (*DSConfluence) SupportOffsetFrom ¶

func (j *DSConfluence) SupportOffsetFrom() bool

SupportOffsetFrom - does DS support resuming from offset?

func (*DSConfluence) UseDefaultMapping ¶

func (j *DSConfluence) UseDefaultMapping(ctx *Ctx, raw bool) bool

UseDefaultMapping - apply MappingNotAnalyzeString for raw/rich (raw=fals/true) index in this DS?

func (*DSConfluence) Validate ¶

func (j *DSConfluence) Validate(ctx *Ctx) (err error)

Validate - is current DS configuration OK?

type DSGerrit ¶

type DSGerrit struct {
	DS                  string
	URL                 string // From DA_GERRIT_URL - gerrit repo path
	SingleOrigin        bool   // From DA_GERRIT_SINGLE_ORIGIN - if you want to store only one gerrit endpoint in the index
	User                string // From DA_GERRIT_USER - gerrit user name
	SSHKey              string // From DA_GERRIT_SSH_KEY - must contain full SSH private key - has higher priority than key path
	SSHKeyPath          string // From DA_GERRIT_SSH_KEY_PATH - path to SSH private key, default GerritDefaultSSHKeyPath '~/.ssh/id_rsa'
	SSHPort             int    // From DA_GERRIT_SSH_PORT, defaults to GerritDefaultSSHPort (29418)
	MaxReviews          int    // From DA_GERRIT_MAX_REVIEWS, defaults to GerritDefaultMaxReviews (1000)
	NoSSLVerify         bool   // From DA_GERRIT_NO_SSL_VERIFY
	DisableHostKeyCheck bool   // From DA_GERRIT_DISABLE_HOST_KEY_CHECK
	// Non-config variables
	SSHOpts        string   // SSH Options
	SSHKeyTempPath string   // if used SSHKey - temp file with this name was used to store key contents
	GerritCmd      []string // gerrit remote command used to fetch data
	VersionMajor   int      // gerrit major version
	VersionMinor   int      // gerrit minor version
}

DSGerrit - DS implementation for stub - does nothing at all, just presents a skeleton code

func (*DSGerrit) AddMetadata ¶

func (j *DSGerrit) AddMetadata(ctx *Ctx, item interface{}) (mItem map[string]interface{})

AddMetadata - add metadata to the item

func (*DSGerrit) AffsItems ¶

func (j *DSGerrit) AffsItems(ctx *Ctx, review map[string]interface{}, roles []string, date interface{}) (affsItems map[string]interface{}, err error)

AffsItems - return affiliations data items for given roles and date

func (*DSGerrit) AllRoles ¶

func (j *DSGerrit) AllRoles(ctx *Ctx, rich map[string]interface{}) (roles []string, static bool)

AllRoles - return all roles defined for the backend roles can be static (always the same) or dynamic (per item) second return parameter is static mode (true/false) dynamic roles will use item to get its roles

func (*DSGerrit) CalculateTimeToReset ¶

func (j *DSGerrit) CalculateTimeToReset(ctx *Ctx, rateLimit, rateLimitReset int) (seconds int)

CalculateTimeToReset - calculate time to reset rate limits based on rate limit value and rate limit reset value

func (*DSGerrit) Categories ¶

func (j *DSGerrit) Categories() map[string]struct{}

Categories - return a set of configured categories

func (*DSGerrit) ConvertDates ¶

func (j *DSGerrit) ConvertDates(ctx *Ctx, review map[string]interface{})

ConvertDates - convert floating point dates to datetimes

func (*DSGerrit) CustomEnrich ¶

func (j *DSGerrit) CustomEnrich() bool

CustomEnrich - is this datasource using custom enrich implementation?

func (*DSGerrit) CustomFetchRaw ¶

func (j *DSGerrit) CustomFetchRaw() bool

CustomFetchRaw - is this datasource using custom fetch raw implementation?

func (*DSGerrit) DateField ¶

func (j *DSGerrit) DateField(*Ctx) string

DateField - return date field used to detect where to restart from

func (*DSGerrit) ElasticRawMapping ¶

func (j *DSGerrit) ElasticRawMapping() []byte

ElasticRawMapping - Raw index mapping definition

func (*DSGerrit) ElasticRichMapping ¶

func (j *DSGerrit) ElasticRichMapping() []byte

ElasticRichMapping - Rich index mapping definition

func (*DSGerrit) Enrich ¶

func (j *DSGerrit) Enrich(ctx *Ctx) (err error)

Enrich - implement enrich data for stub datasource

func (*DSGerrit) EnrichApprovals ¶

func (j *DSGerrit) EnrichApprovals(ctx *Ctx, review, patchSet map[string]interface{}, approvals []map[string]interface{}, affs bool) (richItems []interface{}, err error)

EnrichApprovals - return rich items from raw approvals

func (*DSGerrit) EnrichComments ¶

func (j *DSGerrit) EnrichComments(ctx *Ctx, review map[string]interface{}, comments []map[string]interface{}, affs bool) (richItems []interface{}, err error)

EnrichComments - return rich items from raw patch sets

func (*DSGerrit) EnrichItem ¶

func (j *DSGerrit) EnrichItem(ctx *Ctx, item map[string]interface{}, author string, affs bool, extra interface{}) (rich map[string]interface{}, err error)

EnrichItem - return rich item from raw item

func (*DSGerrit) EnrichItems ¶

func (j *DSGerrit) EnrichItems(ctx *Ctx) (err error)

EnrichItems - perform the enrichment

func (*DSGerrit) EnrichPatchsets ¶

func (j *DSGerrit) EnrichPatchsets(ctx *Ctx, review map[string]interface{}, patchSets []map[string]interface{}, affs bool) (richItems []interface{}, err error)

EnrichPatchsets - return rich items from raw patch sets

func (*DSGerrit) FetchItems ¶

func (j *DSGerrit) FetchItems(ctx *Ctx) (err error)

FetchItems - implement enrich data for stub datasource

func (*DSGerrit) FetchRaw ¶

func (j *DSGerrit) FetchRaw(ctx *Ctx) (err error)

FetchRaw - implement fetch raw data for stub datasource

func (*DSGerrit) FirstApprovalDatetime ¶

func (j *DSGerrit) FirstApprovalDatetime(ctx *Ctx, review map[string]interface{}, patchSets []interface{}) (approvalDatetime interface{})

FirstApprovalDatetime - return first approval date/time

func (*DSGerrit) FirstPatchsetApprovalDatetime ¶

func (j *DSGerrit) FirstPatchsetApprovalDatetime(ctx *Ctx, patchSet map[string]interface{}) (approvalDatetime interface{})

FirstPatchsetApprovalDatetime - return first patchset review date/time

func (*DSGerrit) FirstPatchsetReviewDatetime ¶

func (j *DSGerrit) FirstPatchsetReviewDatetime(ctx *Ctx, patchSet map[string]interface{}) (reviewDatetime interface{})

FirstPatchsetReviewDatetime - return first patchset review date/time

func (*DSGerrit) FirstReviewDatetime ¶

func (j *DSGerrit) FirstReviewDatetime(ctx *Ctx, review map[string]interface{}, patchSets []interface{}) (reviewDatetime interface{})

FirstReviewDatetime - return first review date/time

func (*DSGerrit) GetGerritReviews ¶

func (j *DSGerrit) GetGerritReviews(ctx *Ctx, after string, afterEpoch float64, startFrom int) (reviews []map[string]interface{}, newStartFrom int, err error)

GetGerritReviews - get gerrit reviews

func (*DSGerrit) GetGerritVersion ¶

func (j *DSGerrit) GetGerritVersion(ctx *Ctx) (err error)

GetGerritVersion - get gerrit version

func (*DSGerrit) GetItemIdentities ¶

func (j *DSGerrit) GetItemIdentities(ctx *Ctx, doc interface{}) (identities map[[3]string]struct{}, err error)

GetItemIdentities return list of item's identities, each one is [3]string (name, username, email) tripples, special value Nil "none" means null we use string and not *string which allows nil to allow usage as a map key

func (*DSGerrit) GetRoleIdentity ¶

func (j *DSGerrit) GetRoleIdentity(ctx *Ctx, item map[string]interface{}, role string) (identity map[string]interface{})

GetRoleIdentity - return identity data for a given role

func (*DSGerrit) HasIdentities ¶

func (j *DSGerrit) HasIdentities() bool

HasIdentities - does this data source support identity data

func (*DSGerrit) IdentityForObject ¶

func (j *DSGerrit) IdentityForObject(ctx *Ctx, obj map[string]interface{}) (identity [3]string)

IdentityForObject - construct identity from a given object

func (DSGerrit) Info ¶

func (j DSGerrit) Info() string

Info - return DS configuration in a human readable form

func (*DSGerrit) InitGerrit ¶

func (j *DSGerrit) InitGerrit(ctx *Ctx) (err error)

InitGerrit - initializes gerrit client

func (*DSGerrit) ItemCategory ¶

func (j *DSGerrit) ItemCategory(item interface{}) string

ItemCategory - return unique identifier for an item

func (*DSGerrit) ItemID ¶

func (j *DSGerrit) ItemID(item interface{}) string

ItemID - return unique identifier for an item

func (*DSGerrit) ItemUpdatedOn ¶

func (j *DSGerrit) ItemUpdatedOn(item interface{}) time.Time

ItemUpdatedOn - return updated on date for an item

func (*DSGerrit) LastChangesetApprovalValue ¶

func (j *DSGerrit) LastChangesetApprovalValue(ctx *Ctx, patchSets []interface{}) (status interface{})

LastChangesetApprovalValue - return last approval status

func (*DSGerrit) Name ¶

func (j *DSGerrit) Name() string

Name - return data source name

func (*DSGerrit) OffsetField ¶

func (j *DSGerrit) OffsetField(*Ctx) string

OffsetField - return offset field used to detect where to restart from

func (*DSGerrit) Origin ¶

func (j *DSGerrit) Origin(ctx *Ctx) string

Origin - return current origin

func (*DSGerrit) OriginField ¶

func (j *DSGerrit) OriginField(ctx *Ctx) string

OriginField - return origin field used to detect where to restart from

func (*DSGerrit) ParseArgs ¶

func (j *DSGerrit) ParseArgs(ctx *Ctx) (err error)

ParseArgs - parse gerrit specific environment variables

func (*DSGerrit) ResumeNeedsCategory ¶

func (j *DSGerrit) ResumeNeedsCategory(ctx *Ctx, raw bool) bool

ResumeNeedsCategory - is category field needed when resuming Category should be needed when multiple types of categories save to the same index or there are multiple types of documents within the same category

func (*DSGerrit) ResumeNeedsOrigin ¶

func (j *DSGerrit) ResumeNeedsOrigin(ctx *Ctx, raw bool) bool

ResumeNeedsOrigin - is origin field needed when resuming Origin should be needed when multiple configurations save to the same index

func (*DSGerrit) RichAuthorField ¶

func (j *DSGerrit) RichAuthorField(*Ctx) string

RichAuthorField - return rich ID field name

func (*DSGerrit) RichIDField ¶

func (j *DSGerrit) RichIDField(*Ctx) string

RichIDField - return rich ID field name

func (*DSGerrit) SupportDateFrom ¶

func (j *DSGerrit) SupportDateFrom() bool

SupportDateFrom - does DS support resuming from date?

func (*DSGerrit) SupportOffsetFrom ¶

func (j *DSGerrit) SupportOffsetFrom() bool

SupportOffsetFrom - does DS support resuming from offset?

func (*DSGerrit) UseDefaultMapping ¶

func (j *DSGerrit) UseDefaultMapping(ctx *Ctx, raw bool) bool

UseDefaultMapping - apply MappingNotAnalyzeString for raw/rich (raw=fals/true) index in this DS?

func (*DSGerrit) Validate ¶

func (j *DSGerrit) Validate(ctx *Ctx) (err error)

Validate - is current DS configuration OK?

type DSGit ¶

type DSGit struct {
	DS              string
	URL             string // From DA_GIT_URL - git repo path
	SingleOrigin    bool   // From DA_GIT_SINGLE_ORIGIN - if you want to store only one git endpoint in the index
	ReposPath       string // From DA_GIT_REPOS_PATH - default GitDefaultReposPath
	CachePath       string // From DA_GIT_CACHE_PATH - default GitDefaultCachePath
	NoSSLVerify     bool   // From DA_GIT_NO_SSL_VERIFY
	PairProgramming bool   // From DA_GIT_PAIR_PROGRAMMING
	// Non-config variables
	RepoName        string                            // repo name
	Loc             int                               // lines of code as reported by GitOpsCommand
	Pls             []PLS                             // programming language suppary as reported by GitOpsCommand
	GitPath         string                            // path to git repo clone
	LineScanner     *bufio.Scanner                    // line scanner for git log
	CurrLine        int                               // current line in git log
	ParseState      int                               // 0-init, 1-commit, 2-header, 3-message, 4-file
	Commit          map[string]interface{}            // current parsed commit
	CommitFiles     map[string]map[string]interface{} // current commit's files
	RecentLines     []string                          // recent commit lines
	OrphanedCommits []string                          // orphaned commits SHAs
}

DSGit - DS implementation for git

func (*DSGit) AddMetadata ¶

func (j *DSGit) AddMetadata(ctx *Ctx, item interface{}) (mItem map[string]interface{})

AddMetadata - add metadata to the item

func (*DSGit) AffsItems ¶

func (j *DSGit) AffsItems(ctx *Ctx, commit map[string]interface{}, roles []string, date interface{}) (affsItems map[string]interface{}, err error)

AffsItems - return affiliations data items for given roles and date

func (*DSGit) AllRoles ¶

func (j *DSGit) AllRoles(ctx *Ctx, item map[string]interface{}) (roles []string, static bool)

AllRoles - return all roles defined for the backend roles can be static (always the same) or dynamic (per item) second return parameter is static mode (true/false) dynamic roles will use item to get its roles

func (*DSGit) BuildCommit ¶

func (j *DSGit) BuildCommit(ctx *Ctx) (commit map[string]interface{})

BuildCommit - return commit structure from the current parsed object

func (*DSGit) CalculateTimeToReset ¶

func (j *DSGit) CalculateTimeToReset(ctx *Ctx, rateLimit, rateLimitReset int) (seconds int)

CalculateTimeToReset - calculate time to reset rate limits based on rate limit value and rate limit reset value

func (*DSGit) Categories ¶

func (j *DSGit) Categories() map[string]struct{}

Categories - return a set of configured categories

func (*DSGit) CreateGitRepo ¶

func (j *DSGit) CreateGitRepo(ctx *Ctx) (err error)

CreateGitRepo - clone git repo if needed

func (*DSGit) CustomEnrich ¶

func (j *DSGit) CustomEnrich() bool

CustomEnrich - is this datasource using custom enrich implementation?

func (*DSGit) CustomFetchRaw ¶

func (j *DSGit) CustomFetchRaw() bool

CustomFetchRaw - is this datasource using custom fetch raw implementation?

func (*DSGit) DateField ¶

func (j *DSGit) DateField(*Ctx) string

DateField - return date field used to detect where to restart from

func (*DSGit) ElasticRawMapping ¶

func (j *DSGit) ElasticRawMapping() []byte

ElasticRawMapping - Raw index mapping definition

func (*DSGit) ElasticRichMapping ¶

func (j *DSGit) ElasticRichMapping() []byte

ElasticRichMapping - Rich index mapping definition

func (*DSGit) Enrich ¶

func (j *DSGit) Enrich(ctx *Ctx) (err error)

Enrich - implement enrich data for git datasource

func (*DSGit) EnrichItem ¶

func (j *DSGit) EnrichItem(ctx *Ctx, item map[string]interface{}, skip string, affs bool, extra interface{}) (rich map[string]interface{}, err error)

EnrichItem - return rich item from raw item for a given author type

func (*DSGit) EnrichItems ¶

func (j *DSGit) EnrichItems(ctx *Ctx) (err error)

EnrichItems - perform the enrichment

func (*DSGit) ExtractPrevFileName ¶

func (*DSGit) ExtractPrevFileName(f string) (res string)

ExtractPrevFileName - extracts previous file name (before rename/move etc.)

func (*DSGit) FetchItems ¶

func (j *DSGit) FetchItems(ctx *Ctx) (err error)

FetchItems - implement enrich data for git datasource

func (*DSGit) FetchRaw ¶

func (j *DSGit) FetchRaw(ctx *Ctx) (err error)

FetchRaw - implement fetch raw data for git datasource

func (*DSGit) GetAuthors ¶

func (j *DSGit) GetAuthors(ctx *Ctx, m map[string]string, n map[string][]string) (authors map[string]struct{}, author string)

GetAuthors - parse multiple authors used in pair programming mode

func (*DSGit) GetAuthorsData ¶

func (j *DSGit) GetAuthorsData(ctx *Ctx, doc interface{}, auth string) (authorsMap map[string]struct{}, firstAuthor string)

GetAuthorsData - extract authors data from a given field (this supports pair programming)

func (*DSGit) GetCommitURL ¶

func (j *DSGit) GetCommitURL(origin, hash string) string

GetCommitURL - return git commit URL for a given path and SHA

func (*DSGit) GetGitOps ¶

func (j *DSGit) GetGitOps(ctx *Ctx, thrN int) (ch chan error, err error)

GetGitOps - LOC, lang summary stats

func (*DSGit) GetItemIdentities ¶

func (j *DSGit) GetItemIdentities(ctx *Ctx, doc interface{}) (identities map[[3]string]struct{}, err error)

GetItemIdentities return list of item's identities, each one is [3]string (name, username, email) tripples, special value Nil "none" means null we use string and not *string which allows nil to allow usage as a map key this is for raw items

func (*DSGit) GetOrphanedCommits ¶

func (j *DSGit) GetOrphanedCommits(ctx *Ctx, thrN int) (ch chan error, err error)

GetOrphanedCommits - return data about orphaned commits: commits present in git object storage but not present in rev-list - for example squashed commits

func (*DSGit) GetOtherPPAuthors ¶

func (j *DSGit) GetOtherPPAuthors(ctx *Ctx, doc interface{}) (othersMap map[string]map[string]struct{})

GetOtherPPAuthors - get others authors - possible from fields: Signed-off-by and/or Co-authored-by

func (*DSGit) GetOtherTrailersAuthors ¶

func (j *DSGit) GetOtherTrailersAuthors(ctx *Ctx, doc interface{}) (othersMap map[string]map[[2]string]struct{})

GetOtherTrailersAuthors - get others authors - from other trailers fields (mostly for korg) This works on a raw document

func (*DSGit) GetRepoShortURL ¶

func (j *DSGit) GetRepoShortURL(origin string) (repoShortName string)

GetRepoShortURL - return git commit URL for a given path and SHA

func (*DSGit) GetRoleIdentity ¶

func (j *DSGit) GetRoleIdentity(ctx *Ctx, commit map[string]interface{}, role string) map[string]interface{}

GetRoleIdentity - return identity data for a given role

func (*DSGit) HandleRecentLines ¶

func (j *DSGit) HandleRecentLines(line string)

HandleRecentLines - keep last 30 lines, so we can show them on parser error

func (*DSGit) HasIdentities ¶

func (j *DSGit) HasIdentities() bool

HasIdentities - does this data source support identity data

func (*DSGit) IdentitiesFromGitAuthors ¶

func (j *DSGit) IdentitiesFromGitAuthors(ctx *Ctx, authors map[string]struct{}) (identities map[[3]string]struct{})

IdentitiesFromGitAuthors - construct identities from git authors

func (*DSGit) IdentityFromGitAuthor ¶

func (j *DSGit) IdentityFromGitAuthor(ctx *Ctx, author string) (identity [3]string)

IdentityFromGitAuthor - construct identity from git author

func (DSGit) Info ¶

func (j DSGit) Info() string

Info - return DS configuration in a human readable form

func (*DSGit) ItemCategory ¶

func (j *DSGit) ItemCategory(item interface{}) string

ItemCategory - return unique identifier for an item

func (*DSGit) ItemID ¶

func (j *DSGit) ItemID(item interface{}) string

ItemID - return unique identifier for an item

func (*DSGit) ItemUpdatedOn ¶

func (j *DSGit) ItemUpdatedOn(item interface{}) time.Time

ItemUpdatedOn - return updated on date for an item

func (*DSGit) MarkOrphanedCommits ¶

func (j *DSGit) MarkOrphanedCommits(ctx *Ctx) (err error)

MarkOrphanedCommits - mark all orphaned commits as "orphaned: true"

func (*DSGit) Name ¶

func (j *DSGit) Name() string

Name - return data source name

func (*DSGit) OffsetField ¶

func (j *DSGit) OffsetField(*Ctx) string

OffsetField - return offset field used to detect where to restart from

func (*DSGit) Origin ¶

func (j *DSGit) Origin(ctx *Ctx) string

Origin - return current origin

func (*DSGit) OriginField ¶

func (j *DSGit) OriginField(ctx *Ctx) string

OriginField - return origin field used to detect where to restart from

func (*DSGit) PairProgrammingMetrics ¶

func (j *DSGit) PairProgrammingMetrics(ctx *Ctx, rich, commit map[string]interface{}) (err error)

PairProgrammingMetrics - calculate pair programming metrics data

func (*DSGit) ParseAction ¶

func (j *DSGit) ParseAction(ctx *Ctx, data map[string]string)

ParseAction - parse action line

func (*DSGit) ParseArgs ¶

func (j *DSGit) ParseArgs(ctx *Ctx) (err error)

ParseArgs - parse git specific environment variables

func (*DSGit) ParseCommit ¶

func (j *DSGit) ParseCommit(ctx *Ctx, line string) (parsed bool, err error)

ParseCommit - parse commit

func (*DSGit) ParseFile ¶

func (j *DSGit) ParseFile(ctx *Ctx, line string) (parsed, empty bool, err error)

ParseFile - parse file state

func (*DSGit) ParseGitLog ¶

func (j *DSGit) ParseGitLog(ctx *Ctx) (cmd *exec.Cmd, err error)

ParseGitLog - update git repo

func (*DSGit) ParseHeader ¶

func (j *DSGit) ParseHeader(ctx *Ctx, line string) (parsed bool, err error)

ParseHeader - parse header state

func (*DSGit) ParseInit ¶

func (j *DSGit) ParseInit(ctx *Ctx, line string) (parsed bool, err error)

ParseInit - parse initial state

func (*DSGit) ParseMessage ¶

func (j *DSGit) ParseMessage(ctx *Ctx, line string) (parsed bool, err error)

ParseMessage - parse message state

func (*DSGit) ParseNextCommit ¶

func (j *DSGit) ParseNextCommit(ctx *Ctx) (commit map[string]interface{}, ok bool, err error)

ParseNextCommit - parse next git log commit or report end

func (*DSGit) ParseStats ¶

func (j *DSGit) ParseStats(ctx *Ctx, data map[string]string)

ParseStats - parse stats line

func (*DSGit) ParseTrailer ¶

func (j *DSGit) ParseTrailer(ctx *Ctx, line string)

ParseTrailer - parse possible trailer line

func (*DSGit) ResumeNeedsCategory ¶

func (j *DSGit) ResumeNeedsCategory(ctx *Ctx, raw bool) bool

ResumeNeedsCategory - is category field needed when resuming Category should be needed when multiple types of categories save to the same index or there are multiple types of documents within the same category

func (*DSGit) ResumeNeedsOrigin ¶

func (j *DSGit) ResumeNeedsOrigin(ctx *Ctx, raw bool) bool

ResumeNeedsOrigin - is origin field needed when resuming Origin should be needed when multiple configurations save to the same index

func (*DSGit) RichAuthorField ¶

func (j *DSGit) RichAuthorField(*Ctx) string

RichAuthorField - return rich ID field name

func (*DSGit) RichIDField ¶

func (j *DSGit) RichIDField(*Ctx) string

RichIDField - return rich ID field name

func (*DSGit) SetTLOC ¶

func (j *DSGit) SetTLOC(ctx *Ctx) (err error)

SetTLOC - set total_lines_of_code everywhere in rich index for a current origin

func (*DSGit) SupportDateFrom ¶

func (j *DSGit) SupportDateFrom() bool

SupportDateFrom - does DS support resuming from date?

func (*DSGit) SupportOffsetFrom ¶

func (j *DSGit) SupportOffsetFrom() bool

SupportOffsetFrom - does DS support resuming from offset?

func (*DSGit) TrailerDoc ¶

func (j *DSGit) TrailerDoc(ctx *Ctx, rich, item map[string]interface{}, author string) (trailer map[string]interface{}, skip bool, err error)

TrailerDoc - return flat trailer doc for already generated rich item's nested trailer

func (*DSGit) TrailerDocs ¶

func (j *DSGit) TrailerDocs(ctx *Ctx, rich map[string]interface{}) (trailers []map[string]interface{}, err error)

TrailerDocs - return flat trailer docs for already generated rich item

func (*DSGit) UniqueStringArray ¶

func (j *DSGit) UniqueStringArray(ary []interface{}) []interface{}

UniqueStringArray - make array unique

func (*DSGit) UpdateGitRepo ¶

func (j *DSGit) UpdateGitRepo(ctx *Ctx) (err error)

UpdateGitRepo - update git repo

func (*DSGit) UseDefaultMapping ¶

func (j *DSGit) UseDefaultMapping(ctx *Ctx, raw bool) bool

UseDefaultMapping - apply MappingNotAnalyzeString for raw/rich (raw=fals/true) index in this DS?

func (*DSGit) Validate ¶

func (j *DSGit) Validate(ctx *Ctx) (err error)

Validate - is current DS configuration OK?

type DSGitHub ¶

type DSGitHub struct {
	DS                              string // From DA_DS - data source type "github"
	Org                             string // From DA_GITHUB_ORG - github org
	Repo                            string // From DA_GITHUB_REPO - github repo
	Category                        string // From DA_GITHUB_CATEGORY - issue, pull_request, repository
	Tokens                          string // From DA_GITHUB_TOKENS - "," separated list of OAuth tokens
	URL                             string
	Clients                         []*github.Client
	Context                         context.Context
	OAuthKeys                       []string
	ThrN                            int
	Hint                            int
	RateHandled                     bool
	CanCache                        bool
	CacheDir                        string
	GitHubMtx                       *sync.RWMutex
	GitHubRepoMtx                   *sync.RWMutex
	GitHubIssuesMtx                 *sync.RWMutex
	GitHubUserMtx                   *sync.RWMutex
	GitHubIssueCommentsMtx          *sync.RWMutex
	GitHubCommentReactionsMtx       *sync.RWMutex
	GitHubIssueReactionsMtx         *sync.RWMutex
	GitHubPullMtx                   *sync.RWMutex
	GitHubPullsMtx                  *sync.RWMutex
	GitHubPullReviewsMtx            *sync.RWMutex
	GitHubPullReviewCommentsMtx     *sync.RWMutex
	GitHubReviewCommentReactionsMtx *sync.RWMutex
	GitHubPullRequestedReviewersMtx *sync.RWMutex
	GitHubPullCommitsMtx            *sync.RWMutex
	GitHubUserOrgsMtx               *sync.RWMutex
	GitHubRateMtx                   *sync.RWMutex
	GitHubRepo                      map[string]map[string]interface{}
	GitHubIssues                    map[string][]map[string]interface{}
	GitHubUser                      map[string]map[string]interface{}
	GitHubIssueComments             map[string][]map[string]interface{}
	GitHubCommentReactions          map[string][]map[string]interface{}
	GitHubIssueReactions            map[string][]map[string]interface{}
	GitHubPull                      map[string]map[string]interface{}
	GitHubPulls                     map[string][]map[string]interface{}
	GitHubPullReviews               map[string][]map[string]interface{}
	GitHubPullReviewComments        map[string][]map[string]interface{}
	GitHubReviewCommentReactions    map[string][]map[string]interface{}
	GitHubPullRequestedReviewers    map[string][]map[string]interface{}
	GitHubPullCommits               map[string][]map[string]interface{}
	GitHubUserOrgs                  map[string][]map[string]interface{}
}

DSGitHub - DS implementation for GitHub

func (*DSGitHub) AddMetadata ¶

func (j *DSGitHub) AddMetadata(ctx *Ctx, item interface{}) (mItem map[string]interface{})

AddMetadata - add metadata to the item

func (*DSGitHub) AffsItems ¶

func (j *DSGitHub) AffsItems(ctx *Ctx, item map[string]interface{}, roles []string, date interface{}) (affsItems map[string]interface{}, err error)

AffsItems - return affiliations data items for given roles and date

func (*DSGitHub) AllRoles ¶

func (j *DSGitHub) AllRoles(ctx *Ctx, rich map[string]interface{}) (roles []string, static bool)

AllRoles - return all roles defined for the backend roles can be static (always the same) or dynamic (per item) second return parameter is static mode (true/false) dynamic roles will use item to get its roles

func (*DSGitHub) CalculateTimeToReset ¶

func (j *DSGitHub) CalculateTimeToReset(ctx *Ctx, rateLimit, rateLimitReset int) (seconds int)

CalculateTimeToReset - calculate time to reset rate limits based on rate limit value and rate limit reset value

func (*DSGitHub) Categories ¶

func (j *DSGitHub) Categories() map[string]struct{}

Categories - return a set of configured categories

func (*DSGitHub) CustomEnrich ¶

func (j *DSGitHub) CustomEnrich() bool

CustomEnrich - is this datasource using custom enrich implementation?

func (*DSGitHub) CustomFetchRaw ¶

func (j *DSGitHub) CustomFetchRaw() bool

CustomFetchRaw - is this datasource using custom fetch raw implementation?

func (*DSGitHub) DateField ¶

func (j *DSGitHub) DateField(*Ctx) string

DateField - return date field used to detect where to restart from

func (*DSGitHub) ElasticRawMapping ¶

func (j *DSGitHub) ElasticRawMapping() []byte

ElasticRawMapping - Raw index mapping definition

func (*DSGitHub) ElasticRichMapping ¶

func (j *DSGitHub) ElasticRichMapping() []byte

ElasticRichMapping - Rich index mapping definition

func (*DSGitHub) Enrich ¶

func (j *DSGitHub) Enrich(ctx *Ctx) (err error)

Enrich - implement enrich data for GitHub datasource

func (*DSGitHub) EnrichIssueAssignees ¶

func (j *DSGitHub) EnrichIssueAssignees(ctx *Ctx, issue map[string]interface{}, assignees []map[string]interface{}, affs bool) (richItems []interface{}, err error)

EnrichIssueAssignees - return rich assignees from raw issue

func (*DSGitHub) EnrichIssueComments ¶

func (j *DSGitHub) EnrichIssueComments(ctx *Ctx, issue map[string]interface{}, comments []map[string]interface{}, affs bool) (richItems []interface{}, err error)

EnrichIssueComments - return rich comments from raw issue

func (*DSGitHub) EnrichIssueItem ¶

func (j *DSGitHub) EnrichIssueItem(ctx *Ctx, item map[string]interface{}, author string, affs bool, extra interface{}) (rich map[string]interface{}, err error)

EnrichIssueItem - return rich item from raw item for a given author type

func (*DSGitHub) EnrichIssueReactions ¶

func (j *DSGitHub) EnrichIssueReactions(ctx *Ctx, issue map[string]interface{}, reactions []map[string]interface{}, affs bool) (richItems []interface{}, err error)

EnrichIssueReactions - return rich reactions from raw issue and/or issue comment

func (*DSGitHub) EnrichItem ¶

func (j *DSGitHub) EnrichItem(ctx *Ctx, item map[string]interface{}, author string, affs bool, extra interface{}) (rich map[string]interface{}, err error)

EnrichItem - return rich item from raw item for a given author type

func (*DSGitHub) EnrichItems ¶

func (j *DSGitHub) EnrichItems(ctx *Ctx) (err error)

EnrichItems - perform the enrichment

func (*DSGitHub) EnrichPullRequestAssignees ¶

func (j *DSGitHub) EnrichPullRequestAssignees(ctx *Ctx, pull map[string]interface{}, assignees []map[string]interface{}, affs bool) (richItems []interface{}, err error)

EnrichPullRequestAssignees - return rich assignees from raw pull request

func (*DSGitHub) EnrichPullRequestComments ¶

func (j *DSGitHub) EnrichPullRequestComments(ctx *Ctx, pull map[string]interface{}, comments []map[string]interface{}, affs bool) (richItems []interface{}, err error)

EnrichPullRequestComments - return rich comments from raw pull request

func (*DSGitHub) EnrichPullRequestItem ¶

func (j *DSGitHub) EnrichPullRequestItem(ctx *Ctx, item map[string]interface{}, author string, affs bool, extra interface{}) (rich map[string]interface{}, err error)

EnrichPullRequestItem - return rich item from raw item for a given author type

func (*DSGitHub) EnrichPullRequestReactions ¶

func (j *DSGitHub) EnrichPullRequestReactions(ctx *Ctx, pull map[string]interface{}, reactions []map[string]interface{}, affs bool) (richItems []interface{}, err error)

EnrichPullRequestReactions - return rich reactions from raw pull request comment

func (*DSGitHub) EnrichPullRequestRequestedReviewers ¶

func (j *DSGitHub) EnrichPullRequestRequestedReviewers(ctx *Ctx, pull map[string]interface{}, requestedReviewers []map[string]interface{}, affs bool) (richItems []interface{}, err error)

EnrichPullRequestRequestedReviewers - return rich requested reviewers from raw pull request

func (*DSGitHub) EnrichPullRequestReviews ¶

func (j *DSGitHub) EnrichPullRequestReviews(ctx *Ctx, pull map[string]interface{}, reviews []map[string]interface{}, affs bool) (richItems []interface{}, err error)

EnrichPullRequestReviews - return rich reviews from raw pull request

func (*DSGitHub) EnrichRepositoryItem ¶

func (j *DSGitHub) EnrichRepositoryItem(ctx *Ctx, item map[string]interface{}, author string, affs bool, extra interface{}) (rich map[string]interface{}, err error)

EnrichRepositoryItem - return rich item from raw item for a given author type

func (*DSGitHub) FetchItems ¶

func (j *DSGitHub) FetchItems(ctx *Ctx) (err error)

FetchItems - implement raw data for GitHub datasource

func (*DSGitHub) FetchItemsIssue ¶

func (j *DSGitHub) FetchItemsIssue(ctx *Ctx) (err error)

FetchItemsIssue - implement raw issue data for GitHub datasource

func (*DSGitHub) FetchItemsPullRequest ¶

func (j *DSGitHub) FetchItemsPullRequest(ctx *Ctx) (err error)

FetchItemsPullRequest - implement raw issue data for GitHub datasource

func (*DSGitHub) FetchItemsRepository ¶

func (j *DSGitHub) FetchItemsRepository(ctx *Ctx) (err error)

FetchItemsRepository - implement raw repository data for GitHub datasource

func (*DSGitHub) FetchRaw ¶

func (j *DSGitHub) FetchRaw(ctx *Ctx) (err error)

FetchRaw - implement fetch raw data for GitHub datasource

func (*DSGitHub) GetFirstIssueAttention ¶

func (j *DSGitHub) GetFirstIssueAttention(issue map[string]interface{}) (dt time.Time)

GetFirstIssueAttention - get first non-author action date on the issue

func (*DSGitHub) GetFirstPullRequestReviewDate ¶

func (j *DSGitHub) GetFirstPullRequestReviewDate(pull map[string]interface{}, commsAndReviews bool) (dt time.Time)

GetFirstPullRequestReviewDate - get first review date on a pull request

func (*DSGitHub) GetItemIdentities ¶

func (j *DSGitHub) GetItemIdentities(ctx *Ctx, doc interface{}) (identities map[[3]string]struct{}, err error)

GetItemIdentities return list of item's identities, each one is [3]string (name, username, email) tripples, special value Nil "none" means null we use string and not *string which allows nil to allow usage as a map key

func (*DSGitHub) GetRoleIdentity ¶

func (j *DSGitHub) GetRoleIdentity(ctx *Ctx, item map[string]interface{}, role string) (identity map[string]interface{})

GetRoleIdentity - return identity data for a given role

func (*DSGitHub) GitHubIssueEnrichItemsFunc ¶

func (j *DSGitHub) GitHubIssueEnrichItemsFunc(ctx *Ctx, thrN int, items []interface{}, docs *[]interface{}) (err error)

GitHubIssueEnrichItemsFunc - iterate items and enrich them items is a current pack of input items docs is a pointer to where extracted identities will be stored

func (*DSGitHub) GitHubPullRequestEnrichItemsFunc ¶

func (j *DSGitHub) GitHubPullRequestEnrichItemsFunc(ctx *Ctx, thrN int, items []interface{}, docs *[]interface{}) (err error)

GitHubPullRequestEnrichItemsFunc - iterate items and enrich them items is a current pack of input items docs is a pointer to where extracted identities will be stored

func (*DSGitHub) GitHubRepositoryEnrichItemsFunc ¶

func (j *DSGitHub) GitHubRepositoryEnrichItemsFunc(ctx *Ctx, thrN int, items []interface{}, docs *[]interface{}) (err error)

GitHubRepositoryEnrichItemsFunc - iterate items and enrich them items is a current pack of input items docs is a pointer to where extracted identities will be stored

func (*DSGitHub) HasIdentities ¶

func (j *DSGitHub) HasIdentities() bool

HasIdentities - does this data source support identity data

func (*DSGitHub) IdentityForObject ¶

func (j *DSGitHub) IdentityForObject(ctx *Ctx, item map[string]interface{}) (identity [3]string)

IdentityForObject - construct identity from a given object

func (DSGitHub) Info ¶

func (j DSGitHub) Info() string

Info - return DS configuration in a human readable form

func (*DSGitHub) ItemCategory ¶

func (j *DSGitHub) ItemCategory(item interface{}) string

ItemCategory - return unique identifier for an item

func (*DSGitHub) ItemID ¶

func (j *DSGitHub) ItemID(item interface{}) string

ItemID - return unique identifier for an item

func (*DSGitHub) ItemUpdatedOn ¶

func (j *DSGitHub) ItemUpdatedOn(item interface{}) time.Time

ItemUpdatedOn - return updated on date for an item

func (*DSGitHub) Name ¶

func (j *DSGitHub) Name() string

Name - return data source name

func (*DSGitHub) OffsetField ¶

func (j *DSGitHub) OffsetField(*Ctx) string

OffsetField - return offset field used to detect where to restart from

func (*DSGitHub) Origin ¶

func (j *DSGitHub) Origin(ctx *Ctx) string

Origin - return current origin

func (*DSGitHub) OriginField ¶

func (j *DSGitHub) OriginField(ctx *Ctx) string

OriginField - return origin field used to detect where to restart from

func (*DSGitHub) ParseArgs ¶

func (j *DSGitHub) ParseArgs(ctx *Ctx) (err error)

ParseArgs - parse GitHub specific environment variables

func (*DSGitHub) ProcessIssue ¶

func (j *DSGitHub) ProcessIssue(ctx *Ctx, inIssue map[string]interface{}) (issue map[string]interface{}, err error)

ProcessIssue - add issues sub items

func (*DSGitHub) ProcessPull ¶

func (j *DSGitHub) ProcessPull(ctx *Ctx, inPull map[string]interface{}) (pull map[string]interface{}, err error)

ProcessPull - add PRs sub items

func (*DSGitHub) ResumeNeedsCategory ¶

func (j *DSGitHub) ResumeNeedsCategory(ctx *Ctx, raw bool) bool

ResumeNeedsCategory - is category field needed when resuming Category should be needed when multiple types of categories save to the same index or there are multiple types of documents within the same category

func (*DSGitHub) ResumeNeedsOrigin ¶

func (j *DSGitHub) ResumeNeedsOrigin(ctx *Ctx, raw bool) bool

ResumeNeedsOrigin - is origin field needed when resuming Origin should be needed when multiple configurations save to the same index

func (*DSGitHub) RichAuthorField ¶

func (j *DSGitHub) RichAuthorField(*Ctx) string

RichAuthorField - return rich author field name

func (*DSGitHub) RichIDField ¶

func (j *DSGitHub) RichIDField(*Ctx) string

RichIDField - return rich ID field name

func (*DSGitHub) SupportDateFrom ¶

func (j *DSGitHub) SupportDateFrom() bool

SupportDateFrom - does DS support resuming from date?

func (*DSGitHub) SupportOffsetFrom ¶

func (j *DSGitHub) SupportOffsetFrom() bool

SupportOffsetFrom - does DS support resuming from offset?

func (*DSGitHub) UseDefaultMapping ¶

func (j *DSGitHub) UseDefaultMapping(ctx *Ctx, raw bool) bool

UseDefaultMapping - apply MappingNotAnalyzeString for raw/rich (raw=fals/true) index in this DS?

func (*DSGitHub) Validate ¶

func (j *DSGitHub) Validate(ctx *Ctx) (err error)

Validate - is current DS configuration OK?

type DSGroupsio ¶

type DSGroupsio struct {
	DS           string
	GroupName    string // From DA_GROUPSIO_URL - Group name like GROUP-topic
	Email        string // From DA_GROUPSIO_EMAIL
	Password     string // From DA_GROUPSIO_PASSWORD
	NoSSLVerify  bool   // From DA_GROUPSIO_NO_SSL_VERIFY
	SaveArchives bool   // From DA_GROUPSIO_SAVE_ARCHIVES
	ArchPath     string // From DA_GROUPSIO_ARCH_PATH - default GroupsioDefaultArchPath
	MultiOrigin  bool   // From DA_GROUPSIO_MULTI_ORIGIN - allow multiple groups in a single index
}

DSGroupsio - DS implementation for stub - does nothing at all, just presents a skeleton code

func (*DSGroupsio) AddMetadata ¶

func (j *DSGroupsio) AddMetadata(ctx *Ctx, msg interface{}) (mItem map[string]interface{})

AddMetadata - add metadata to the item

func (*DSGroupsio) AffsItems ¶

func (j *DSGroupsio) AffsItems(ctx *Ctx, rawItem map[string]interface{}, roles []string, date interface{}) (affsItems map[string]interface{}, err error)

AffsItems - return affiliations data items for given roles and date

func (*DSGroupsio) AllRoles ¶

func (j *DSGroupsio) AllRoles(ctx *Ctx, rich map[string]interface{}) (roles []string, static bool)

AllRoles - return all roles defined for Groupsio backend roles can be static (always the same) or dynamic (per item) second return parameter is static mode (true/false) dynamic roles will use item to get its roles

func (*DSGroupsio) CalculateTimeToReset ¶

func (j *DSGroupsio) CalculateTimeToReset(ctx *Ctx, rateLimit, rateLimitReset int) (seconds int)

CalculateTimeToReset - calculate time to reset rate limits based on rate limit value and rate limit reset value

func (*DSGroupsio) Categories ¶

func (j *DSGroupsio) Categories() map[string]struct{}

Categories - return a set of configured categories

func (*DSGroupsio) CustomEnrich ¶

func (j *DSGroupsio) CustomEnrich() bool

CustomEnrich - is this datasource using custom enrich implementation?

func (*DSGroupsio) CustomFetchRaw ¶

func (j *DSGroupsio) CustomFetchRaw() bool

CustomFetchRaw - is this datasource using custom fetch raw implementation?

func (*DSGroupsio) DateField ¶

func (j *DSGroupsio) DateField(*Ctx) string

DateField - return date field used to detect where to restart from

func (*DSGroupsio) ElasticRawMapping ¶

func (j *DSGroupsio) ElasticRawMapping() []byte

ElasticRawMapping - Raw index mapping definition

func (*DSGroupsio) ElasticRichMapping ¶

func (j *DSGroupsio) ElasticRichMapping() []byte

ElasticRichMapping - Rich index mapping definition

func (*DSGroupsio) Enrich ¶

func (j *DSGroupsio) Enrich(ctx *Ctx) (err error)

Enrich - implement enrich data for stub datasource

func (*DSGroupsio) EnrichItem ¶

func (j *DSGroupsio) EnrichItem(ctx *Ctx, item map[string]interface{}, role string, affs bool, extra interface{}) (rich map[string]interface{}, err error)

EnrichItem - return rich item from raw item for a given author type/role

func (*DSGroupsio) EnrichItems ¶

func (j *DSGroupsio) EnrichItems(ctx *Ctx) (err error)

EnrichItems - perform the enrichment

func (*DSGroupsio) FetchItems ¶

func (j *DSGroupsio) FetchItems(ctx *Ctx) (err error)

FetchItems - implement enrich data for stub datasource

func (*DSGroupsio) FetchRaw ¶

func (j *DSGroupsio) FetchRaw(ctx *Ctx) (err error)

FetchRaw - implement fetch raw data for stub datasource

func (*DSGroupsio) GetItemIdentities ¶

func (j *DSGroupsio) GetItemIdentities(ctx *Ctx, doc interface{}) (identities map[[3]string]struct{}, err error)

GetItemIdentities return list of item's identities, each one is [3]string (name, username, email) tripples, special value Nil "none" means null we use string and not *string which allows nil to allow usage as a map key

func (*DSGroupsio) GetItemIdentitiesEx ¶

func (j *DSGroupsio) GetItemIdentitiesEx(ctx *Ctx, doc interface{}) (identities map[[3]string]map[string]struct{}, nRecipients int)

GetItemIdentitiesEx return list of item's identities, each one is [3]string (name, username, email) tripples, special value Nil "none" means null we use string and not *string which allows nil to allow usage as a map key This one (Ex) also returns information about identity's origins (from, to, or both)

func (*DSGroupsio) GetRoleIdentity ¶

func (j *DSGroupsio) GetRoleIdentity(ctx *Ctx, item map[string]interface{}, role string) map[string]interface{}

GetRoleIdentity - return identity data for a given role groupsio si not using this

func (*DSGroupsio) HasIdentities ¶

func (j *DSGroupsio) HasIdentities() bool

HasIdentities - does this data source support identity data

func (DSGroupsio) Info ¶

func (j DSGroupsio) Info() string

Info - return DS configuration in a human readable form

func (*DSGroupsio) ItemCategory ¶

func (j *DSGroupsio) ItemCategory(item interface{}) string

ItemCategory - return unique identifier for an item

func (*DSGroupsio) ItemID ¶

func (j *DSGroupsio) ItemID(item interface{}) string

ItemID - return unique identifier for an item

func (*DSGroupsio) ItemUpdatedOn ¶

func (j *DSGroupsio) ItemUpdatedOn(item interface{}) time.Time

ItemUpdatedOn - return updated on date for an item

func (*DSGroupsio) Name ¶

func (j *DSGroupsio) Name() string

Name - return data source name

func (*DSGroupsio) OffsetField ¶

func (j *DSGroupsio) OffsetField(*Ctx) string

OffsetField - return offset field used to detect where to restart from

func (*DSGroupsio) Origin ¶

func (j *DSGroupsio) Origin(ctx *Ctx) string

Origin - return current origin

func (*DSGroupsio) OriginField ¶

func (j *DSGroupsio) OriginField(ctx *Ctx) string

OriginField - return origin field used to detect where to restart from

func (*DSGroupsio) ParseArgs ¶

func (j *DSGroupsio) ParseArgs(ctx *Ctx) (err error)

ParseArgs - parse stub specific environment variables

func (*DSGroupsio) ResumeNeedsCategory ¶

func (j *DSGroupsio) ResumeNeedsCategory(ctx *Ctx, raw bool) bool

ResumeNeedsCategory - is category field needed when resuming Category should be needed when multiple types of categories save to the same index or there are multiple types of documents within the same category

func (*DSGroupsio) ResumeNeedsOrigin ¶

func (j *DSGroupsio) ResumeNeedsOrigin(ctx *Ctx, raw bool) bool

ResumeNeedsOrigin - is origin field needed when resuming Origin should be needed when multiple configurations save to the same index

func (*DSGroupsio) RichAuthorField ¶

func (j *DSGroupsio) RichAuthorField(*Ctx) string

RichAuthorField - return rich author field name

func (*DSGroupsio) RichIDField ¶

func (j *DSGroupsio) RichIDField(*Ctx) string

RichIDField - return rich ID field name

func (*DSGroupsio) SupportDateFrom ¶

func (j *DSGroupsio) SupportDateFrom() bool

SupportDateFrom - does DS support resuming from date?

func (*DSGroupsio) SupportOffsetFrom ¶

func (j *DSGroupsio) SupportOffsetFrom() bool

SupportOffsetFrom - does DS support resuming from offset?

func (*DSGroupsio) UseDefaultMapping ¶

func (j *DSGroupsio) UseDefaultMapping(ctx *Ctx, raw bool) bool

UseDefaultMapping - apply MappingNotAnalyzeString for raw/rich (raw=fals/true) index in this DS?

func (*DSGroupsio) Validate ¶

func (j *DSGroupsio) Validate(ctx *Ctx) (err error)

Validate - is current DS configuration OK?

type DSJira ¶

type DSJira struct {
	DS          string
	URL         string // From DA_JIRA_URL - Jira URL
	NoSSLVerify bool   // From DA_JIRA_NO_SSL_VERIFY
	User        string // From DA_JIRA_USER - if user is provided then we assume that we don't have base64 encoded user:token yet
	Token       string // From DA_JIRA_TOKEN - if user is not specified we assume that token already contains "<username>:<your-api-token>"
	PageSize    int    // From DA_JIRA_PAGE_SIZE
	MultiOrigin bool   // From DA_JIRA_MULTI_ORIGIN
}

DSJira - DS implementation for Jira

func (*DSJira) AddMetadata ¶

func (j *DSJira) AddMetadata(ctx *Ctx, issue interface{}) (mItem map[string]interface{})

AddMetadata - add metadata to the item

func (*DSJira) AffsItems ¶

func (j *DSJira) AffsItems(ctx *Ctx, item map[string]interface{}, roles []string, date interface{}) (affsItems map[string]interface{}, err error)

AffsItems - return affiliations data items for given roles and date

func (*DSJira) AllRoles ¶

func (j *DSJira) AllRoles(ctx *Ctx, item map[string]interface{}) ([]string, bool)

AllRoles - return all roles defined for Jira backend roles can be static (always the same) or dynamic (per item) second return parameter is static mode (true/false) dynamic roles will use item to get its roles

func (*DSJira) CalculateTimeToReset ¶

func (j *DSJira) CalculateTimeToReset(ctx *Ctx, rateLimit, rateLimitReset int) (seconds int)

CalculateTimeToReset - calculate time to reset rate limits based on rate limit value and rate limit reset value

func (*DSJira) Categories ¶

func (j *DSJira) Categories() map[string]struct{}

Categories - return a set of configured categories

func (*DSJira) CustomEnrich ¶

func (j *DSJira) CustomEnrich() bool

CustomEnrich - is this datasource using custom enrich implementation?

func (*DSJira) CustomFetchRaw ¶

func (j *DSJira) CustomFetchRaw() bool

CustomFetchRaw - is this datasource using custom fetch raw implementation?

func (*DSJira) DateField ¶

func (j *DSJira) DateField(*Ctx) string

DateField - return date field used to detect where to restart from

func (*DSJira) ElasticRawMapping ¶

func (j *DSJira) ElasticRawMapping() []byte

ElasticRawMapping - Raw index mapping definition

func (*DSJira) ElasticRichMapping ¶

func (j *DSJira) ElasticRichMapping() []byte

ElasticRichMapping - Rich index mapping definition

func (*DSJira) Enrich ¶

func (j *DSJira) Enrich(ctx *Ctx) (err error)

Enrich - implement enrich data for Jira

func (*DSJira) EnrichItem ¶

func (j *DSJira) EnrichItem(ctx *Ctx, item map[string]interface{}, author string, affs bool, extra interface{}) (rich map[string]interface{}, err error)

EnrichItem - return rich item from raw item for a given author type

func (*DSJira) EnrichItems ¶

func (j *DSJira) EnrichItems(ctx *Ctx) (err error)

EnrichItems - perform the enrichment

func (*DSJira) FetchItems ¶

func (j *DSJira) FetchItems(ctx *Ctx) (err error)

FetchItems - implement fetch items for jira datasource

func (*DSJira) FetchRaw ¶

func (j *DSJira) FetchRaw(ctx *Ctx) (err error)

FetchRaw - implement fetch raw data for Jira

func (*DSJira) GenSearchFields ¶

func (j *DSJira) GenSearchFields(ctx *Ctx, issue interface{}, uuid string) (fields map[string]interface{})

GenSearchFields - generate extra search fields

func (*DSJira) GetFields ¶

func (j *DSJira) GetFields(ctx *Ctx) (customFields map[string]JiraField, err error)

GetFields - implement get fields for jira datasource

func (*DSJira) GetItemIdentities ¶

func (j *DSJira) GetItemIdentities(ctx *Ctx, doc interface{}) (identities map[[3]string]struct{}, err error)

GetItemIdentities return list of item's identities, each one is [3]string (name, username, email) tripples, special value Nil "none" means null we use string and not *string which allows nil to allow usage as a map key

func (*DSJira) GetRoleIdentity ¶

func (j *DSJira) GetRoleIdentity(ctx *Ctx, item map[string]interface{}, role string) (identity map[string]interface{})

GetRoleIdentity - return identity data for a given role

func (*DSJira) HasIdentities ¶

func (j *DSJira) HasIdentities() bool

HasIdentities - does this data source support identity data

func (DSJira) Info ¶

func (j DSJira) Info() string

Info - return DS configuration in a human readable form

func (*DSJira) ItemCategory ¶

func (j *DSJira) ItemCategory(item interface{}) string

ItemCategory - return unique identifier for an item

func (*DSJira) ItemID ¶

func (j *DSJira) ItemID(item interface{}) string

ItemID - return unique identifier for an item

func (*DSJira) ItemUpdatedOn ¶

func (j *DSJira) ItemUpdatedOn(item interface{}) time.Time

ItemUpdatedOn - return updated on date for an item

func (*DSJira) Name ¶

func (j *DSJira) Name() string

Name - return data source name

func (*DSJira) OffsetField ¶

func (j *DSJira) OffsetField(*Ctx) string

OffsetField - return offset field used to detect where to restart from

func (*DSJira) Origin ¶

func (j *DSJira) Origin(ctx *Ctx) string

Origin - return current origin Tag gets precendence if set

func (*DSJira) OriginField ¶

func (j *DSJira) OriginField(ctx *Ctx) string

OriginField - return origin field used to detect where to restart from

func (*DSJira) ParseArgs ¶

func (j *DSJira) ParseArgs(ctx *Ctx) (err error)

ParseArgs - parse jira specific environment variables

func (*DSJira) ProcessIssue ¶

func (j *DSJira) ProcessIssue(ctx *Ctx, allIssues *[]interface{}, allIssuesMtx *sync.Mutex, issue interface{}, customFields map[string]JiraField, from time.Time, to *time.Time, thrN int) (wch chan error, err error)

ProcessIssue - process a single issue

func (*DSJira) ResumeNeedsCategory ¶

func (j *DSJira) ResumeNeedsCategory(ctx *Ctx, raw bool) bool

ResumeNeedsCategory - is category field needed when resuming Category should be needed when multiple types of categories save to the same index or there are multiple types of documents within the same category

func (*DSJira) ResumeNeedsOrigin ¶

func (j *DSJira) ResumeNeedsOrigin(ctx *Ctx, raw bool) bool

ResumeNeedsOrigin - is origin field needed when resuming Origin should be needed when multiple configurations save to the same index Jira usually stores only one instance per index, so we don't need to enable filtering by origin to resume

func (*DSJira) RichAuthorField ¶

func (j *DSJira) RichAuthorField(*Ctx) string

RichAuthorField - return rich author field name

func (*DSJira) RichIDField ¶

func (j *DSJira) RichIDField(*Ctx) string

RichIDField - return rich ID field name

func (*DSJira) SearchFields ¶

func (j *DSJira) SearchFields() map[string][]string

SearchFields - define (optional) search fields to be returned

func (*DSJira) SupportDateFrom ¶

func (j *DSJira) SupportDateFrom() bool

SupportDateFrom - does DS support resuming from date?

func (*DSJira) SupportOffsetFrom ¶

func (j *DSJira) SupportOffsetFrom() bool

SupportOffsetFrom - does DS support resuming from offset?

func (*DSJira) UseDefaultMapping ¶

func (j *DSJira) UseDefaultMapping(ctx *Ctx, raw bool) bool

UseDefaultMapping - apply MappingNotAnalyzeString for raw/rich (raw=fals/true) index in this DS?

func (*DSJira) Validate ¶

func (j *DSJira) Validate(ctx *Ctx) (err error)

Validate - is current DS configuration OK?

type DSRocketchat ¶

type DSRocketchat struct {
	DS           string
	URL          string // From DA_ROCKETCHAT_URL - rocketchat server url
	Channel      string // From DA_ROCKETCHAT_CHANNEL - rocketchat channel
	User         string // From DA_ROCKETCHAT_USER - user name
	Token        string // From DA_ROCKETCHAT_TOKEN - token
	MaxItems     int    // From DA_ROCKETCHAT_MAX_ITEMS, defaults to RocketchatDefaultMaxItems (100)
	MinRate      int    // From DA_ROCKETCHAT_MIN_RATE - min API points, if we reach this value we wait for refresh, default RocketchatDefaultMinRate(10)
	WaitRate     bool   // From DA_ROCKETCHAT_WAIT_RATE - will wait for rate limit refresh if set, otherwise will fail is rate limit is reached
	NoSSLVerify  bool   // From DA_ROCKETCHAT_NO_SSL_VERIFY
	SingleOrigin bool   // From DA_ROCKETCHAT_SINGLE_ORIGIN - if you want to store only one rocketchat endpoint in the index
}

DSRocketchat - DS implementation for rocketchat - does nothing at all, just presents a skeleton code

func (*DSRocketchat) AddMetadata ¶

func (j *DSRocketchat) AddMetadata(ctx *Ctx, item interface{}) (mItem map[string]interface{})

AddMetadata - add metadata to the item

func (*DSRocketchat) AffsItems ¶

func (j *DSRocketchat) AffsItems(ctx *Ctx, message map[string]interface{}, roles []string, date interface{}) (affsItems map[string]interface{}, err error)

AffsItems - return affiliations data items for given roles and date

func (*DSRocketchat) AllRoles ¶

func (j *DSRocketchat) AllRoles(ctx *Ctx, item map[string]interface{}) ([]string, bool)

AllRoles - return all roles defined for the backend roles can be static (always the same) or dynamic (per item) second return parameter is static mode (true/false) dynamic roles will use item to get its roles

func (*DSRocketchat) CalculateTimeToReset ¶

func (j *DSRocketchat) CalculateTimeToReset(ctx *Ctx, rateLimit, rateLimitReset int) (seconds int)

CalculateTimeToReset - calculate time to reset rate limits based on rate limit value and rate limit reset value

func (*DSRocketchat) Categories ¶

func (j *DSRocketchat) Categories() map[string]struct{}

Categories - return a set of configured categories

func (*DSRocketchat) CustomEnrich ¶

func (j *DSRocketchat) CustomEnrich() bool

CustomEnrich - is this datasource using custom enrich implementation?

func (*DSRocketchat) CustomFetchRaw ¶

func (j *DSRocketchat) CustomFetchRaw() bool

CustomFetchRaw - is this datasource using custom fetch raw implementation?

func (*DSRocketchat) DateField ¶

func (j *DSRocketchat) DateField(*Ctx) string

DateField - return date field used to detect where to restart from

func (*DSRocketchat) ElasticRawMapping ¶

func (j *DSRocketchat) ElasticRawMapping() []byte

ElasticRawMapping - Raw index mapping definition

func (*DSRocketchat) ElasticRichMapping ¶

func (j *DSRocketchat) ElasticRichMapping() []byte

ElasticRichMapping - Rich index mapping definition

func (*DSRocketchat) Enrich ¶

func (j *DSRocketchat) Enrich(ctx *Ctx) (err error)

Enrich - implement enrich data for rocketchat datasource

func (*DSRocketchat) EnrichItem ¶

func (j *DSRocketchat) EnrichItem(ctx *Ctx, item map[string]interface{}, author string, affs bool, extra interface{}) (rich map[string]interface{}, err error)

EnrichItem - return rich item from raw item for a given author type

func (*DSRocketchat) EnrichItems ¶

func (j *DSRocketchat) EnrichItems(ctx *Ctx) (err error)

EnrichItems - perform the enrichment

func (*DSRocketchat) FetchItems ¶

func (j *DSRocketchat) FetchItems(ctx *Ctx) (err error)

FetchItems - implement enrich data for rocketchat datasource

func (*DSRocketchat) FetchRaw ¶

func (j *DSRocketchat) FetchRaw(ctx *Ctx) (err error)

FetchRaw - implement fetch raw data for rocketchat datasource

func (*DSRocketchat) GetItemIdentities ¶

func (j *DSRocketchat) GetItemIdentities(ctx *Ctx, doc interface{}) (identities map[[3]string]struct{}, err error)

GetItemIdentities return list of item's identities, each one is [3]string (name, username, email) tripples, special value Nil "none" means null we use string and not *string which allows nil to allow usage as a map key

func (*DSRocketchat) GetMentions ¶

func (j *DSRocketchat) GetMentions(mentions []interface{}) (richMentions []map[string]interface{})

GetMentions - convert raw mentions to rich mentions

func (*DSRocketchat) GetReactions ¶

func (j *DSRocketchat) GetReactions(reactions map[string]interface{}) (richReactions []map[string]interface{}, nReactions int)

GetReactions - convert raw reactions to rich reactions

func (*DSRocketchat) GetRocketchatMessages ¶

func (j *DSRocketchat) GetRocketchatMessages(ctx *Ctx, fromDate string, offset, rateLimit, rateLimitReset, thrN int) (messages []map[string]interface{}, newOffset, total, outRateLimit, outRateLimitReset int, err error)

GetRocketchatMessages - get confluence historical contents

func (*DSRocketchat) GetRoleIdentity ¶

func (j *DSRocketchat) GetRoleIdentity(ctx *Ctx, item map[string]interface{}, role string) (identity map[string]interface{})

GetRoleIdentity - return identity data for a given role

func (*DSRocketchat) HasIdentities ¶

func (j *DSRocketchat) HasIdentities() bool

HasIdentities - does this data source support identity data

func (DSRocketchat) Info ¶

func (j DSRocketchat) Info() string

Info - return DS configuration in a human readable form

func (*DSRocketchat) ItemCategory ¶

func (j *DSRocketchat) ItemCategory(item interface{}) string

ItemCategory - return unique identifier for an item

func (*DSRocketchat) ItemID ¶

func (j *DSRocketchat) ItemID(item interface{}) string

ItemID - return unique identifier for an item

func (*DSRocketchat) ItemUpdatedOn ¶

func (j *DSRocketchat) ItemUpdatedOn(item interface{}) time.Time

ItemUpdatedOn - return updated on date for an item

func (*DSRocketchat) Name ¶

func (j *DSRocketchat) Name() string

Name - return data source name

func (*DSRocketchat) OffsetField ¶

func (j *DSRocketchat) OffsetField(*Ctx) string

OffsetField - return offset field used to detect where to restart from

func (*DSRocketchat) Origin ¶

func (j *DSRocketchat) Origin(ctx *Ctx) string

Origin - return current origin

func (*DSRocketchat) OriginField ¶

func (j *DSRocketchat) OriginField(ctx *Ctx) string

OriginField - return origin field used to detect where to restart from

func (*DSRocketchat) ParseArgs ¶

func (j *DSRocketchat) ParseArgs(ctx *Ctx) (err error)

ParseArgs - parse rocketchat specific environment variables

func (*DSRocketchat) ResumeNeedsCategory ¶

func (j *DSRocketchat) ResumeNeedsCategory(ctx *Ctx, raw bool) bool

ResumeNeedsCategory - is category field needed when resuming Category should be needed when multiple types of categories save to the same index or there are multiple types of documents within the same category

func (*DSRocketchat) ResumeNeedsOrigin ¶

func (j *DSRocketchat) ResumeNeedsOrigin(ctx *Ctx, raw bool) bool

ResumeNeedsOrigin - is origin field needed when resuming Origin should be needed when multiple configurations save to the same index

func (*DSRocketchat) RichAuthorField ¶

func (j *DSRocketchat) RichAuthorField(*Ctx) string

RichAuthorField - return rich author field name

func (*DSRocketchat) RichIDField ¶

func (j *DSRocketchat) RichIDField(*Ctx) string

RichIDField - return rich ID field name

func (*DSRocketchat) SetChannelInfo ¶

func (j *DSRocketchat) SetChannelInfo(rich, channel map[string]interface{})

SetChannelInfo - set rich channel info from raw channel info

func (*DSRocketchat) SleepAsRequested ¶

func (j *DSRocketchat) SleepAsRequested(res interface{}, thrN int)

SleepAsRequested - parse server's: {"success":false,"error":"Error, too many requests. Please slow down. You must wait 23 seconds before trying this endpoint again. [error-too-many-requests]"} And sleep N+1 requested seconds

func (*DSRocketchat) SupportDateFrom ¶

func (j *DSRocketchat) SupportDateFrom() bool

SupportDateFrom - does DS support resuming from date?

func (*DSRocketchat) SupportOffsetFrom ¶

func (j *DSRocketchat) SupportOffsetFrom() bool

SupportOffsetFrom - does DS support resuming from offset?

func (*DSRocketchat) UseDefaultMapping ¶

func (j *DSRocketchat) UseDefaultMapping(ctx *Ctx, raw bool) bool

UseDefaultMapping - apply MappingNotAnalyzeString for raw/rich (raw=fals/true) index in this DS?

func (*DSRocketchat) Validate ¶

func (j *DSRocketchat) Validate(ctx *Ctx) (err error)

Validate - is current DS configuration OK?

type DSStub ¶

type DSStub struct {
	DS          string
	MultiOrigin bool // can we store multiple endpoints in a single index?
}

DSStub - DS implementation for stub - does nothing at all, just presents a skeleton code

func (*DSStub) AddMetadata ¶

func (j *DSStub) AddMetadata(ctx *Ctx, item interface{}) (mItem map[string]interface{})

AddMetadata - add metadata to the item

func (*DSStub) AffsItems ¶

func (j *DSStub) AffsItems(ctx *Ctx, rawItem map[string]interface{}, roles []string, date interface{}) (affsItems map[string]interface{}, err error)

AffsItems - return affiliations data items for given roles and date

func (*DSStub) AllRoles ¶

func (j *DSStub) AllRoles(ctx *Ctx, item map[string]interface{}) ([]string, bool)

AllRoles - return all roles defined for the backend roles can be static (always the same) or dynamic (per item) second return parameter is static mode (true/false) dynamic roles will use item to get its roles

func (*DSStub) CalculateTimeToReset ¶

func (j *DSStub) CalculateTimeToReset(ctx *Ctx, rateLimit, rateLimitReset int) (seconds int)

CalculateTimeToReset - calculate time to reset rate limits based on rate limit value and rate limit reset value

func (*DSStub) Categories ¶

func (j *DSStub) Categories() map[string]struct{}

Categories - return a set of configured categories

func (*DSStub) CustomEnrich ¶

func (j *DSStub) CustomEnrich() bool

CustomEnrich - is this datasource using custom enrich implementation?

func (*DSStub) CustomFetchRaw ¶

func (j *DSStub) CustomFetchRaw() bool

CustomFetchRaw - is this datasource using custom fetch raw implementation?

func (*DSStub) DateField ¶

func (j *DSStub) DateField(*Ctx) string

DateField - return date field used to detect where to restart from

func (*DSStub) ElasticRawMapping ¶

func (j *DSStub) ElasticRawMapping() []byte

ElasticRawMapping - Raw index mapping definition

func (*DSStub) ElasticRichMapping ¶

func (j *DSStub) ElasticRichMapping() []byte

ElasticRichMapping - Rich index mapping definition

func (*DSStub) Enrich ¶

func (j *DSStub) Enrich(ctx *Ctx) (err error)

Enrich - implement enrich data for stub datasource

func (*DSStub) EnrichItem ¶

func (j *DSStub) EnrichItem(ctx *Ctx, item map[string]interface{}, author string, affs bool, extra interface{}) (rich map[string]interface{}, err error)

EnrichItem - return rich item from raw item for a given author type

func (*DSStub) EnrichItems ¶

func (j *DSStub) EnrichItems(ctx *Ctx) (err error)

EnrichItems - perform the enrichment

func (*DSStub) FetchItems ¶

func (j *DSStub) FetchItems(ctx *Ctx) (err error)

FetchItems - implement enrich data for stub datasource

func (*DSStub) FetchRaw ¶

func (j *DSStub) FetchRaw(ctx *Ctx) (err error)

FetchRaw - implement fetch raw data for stub datasource

func (*DSStub) GetItemIdentities ¶

func (j *DSStub) GetItemIdentities(ctx *Ctx, doc interface{}) (map[[3]string]struct{}, error)

GetItemIdentities return list of item's identities, each one is [3]string (name, username, email) tripples, special value Nil "none" means null we use string and not *string which allows nil to allow usage as a map key

func (*DSStub) GetRoleIdentity ¶

func (j *DSStub) GetRoleIdentity(ctx *Ctx, item map[string]interface{}, role string) map[string]interface{}

GetRoleIdentity - return identity data for a given role

func (*DSStub) HasIdentities ¶

func (j *DSStub) HasIdentities() bool

HasIdentities - does this data source support identity data

func (DSStub) Info ¶

func (j DSStub) Info() string

Info - return DS configuration in a human readable form

func (*DSStub) ItemCategory ¶

func (j *DSStub) ItemCategory(item interface{}) string

ItemCategory - return unique identifier for an item

func (*DSStub) ItemID ¶

func (j *DSStub) ItemID(item interface{}) string

ItemID - return unique identifier for an item

func (*DSStub) ItemUpdatedOn ¶

func (j *DSStub) ItemUpdatedOn(item interface{}) time.Time

ItemUpdatedOn - return updated on date for an item

func (*DSStub) Name ¶

func (j *DSStub) Name() string

Name - return data source name

func (*DSStub) OffsetField ¶

func (j *DSStub) OffsetField(*Ctx) string

OffsetField - return offset field used to detect where to restart from

func (*DSStub) Origin ¶

func (j *DSStub) Origin(ctx *Ctx) string

Origin - return current origin

func (*DSStub) OriginField ¶

func (j *DSStub) OriginField(ctx *Ctx) string

OriginField - return origin field used to detect where to restart from

func (*DSStub) ParseArgs ¶

func (j *DSStub) ParseArgs(ctx *Ctx) (err error)

ParseArgs - parse stub specific environment variables

func (*DSStub) ResumeNeedsCategory ¶

func (j *DSStub) ResumeNeedsCategory(ctx *Ctx, raw bool) bool

ResumeNeedsCategory - is category field needed when resuming Category should be needed when multiple types of categories save to the same index or there are multiple types of documents within the same category

func (*DSStub) ResumeNeedsOrigin ¶

func (j *DSStub) ResumeNeedsOrigin(ctx *Ctx, raw bool) bool

ResumeNeedsOrigin - is origin field needed when resuming Origin should be needed when multiple configurations save to the same index

func (*DSStub) RichAuthorField ¶

func (j *DSStub) RichAuthorField(*Ctx) string

RichAuthorField - return rich author field name

func (*DSStub) RichIDField ¶

func (j *DSStub) RichIDField(*Ctx) string

RichIDField - return rich ID field name

func (*DSStub) SupportDateFrom ¶

func (j *DSStub) SupportDateFrom() bool

SupportDateFrom - does DS support resuming from date?

func (*DSStub) SupportOffsetFrom ¶

func (j *DSStub) SupportOffsetFrom() bool

SupportOffsetFrom - does DS support resuming from offset?

func (*DSStub) UseDefaultMapping ¶

func (j *DSStub) UseDefaultMapping(ctx *Ctx, raw bool) bool

UseDefaultMapping - apply MappingNotAnalyzeString for raw/rich (raw=fals/true) index in this DS?

func (*DSStub) Validate ¶

func (j *DSStub) Validate(ctx *Ctx) (err error)

Validate - is current DS configuration OK?

type DateCacheEntry ¶

type DateCacheEntry struct {
	Dt     time.Time
	DtInTz time.Time
	TzOff  float64
	Valid  bool
}

DateCacheEntry - parse date cache entry

type ESCacheEntry ¶

type ESCacheEntry struct {
	K string    `json:"k"` // cache key
	G string    `json:"g"` // cache tag
	B []byte    `json:"b"` // cache data
	T time.Time `json:"t"` // when cached
	E time.Time `json:"e"` // when expires
}

ESCacheEntry - single cache entry

func ESCacheGet ¶

func ESCacheGet(ctx *Ctx, key string) (entry *ESCacheEntry, ok bool)

ESCacheGet - get value from cache

type Flag ¶

type Flag string

Flag gets CLI flag values

func NewFlag ¶

func NewFlag() *Flag

NewFlag ...

func (*Flag) Bool ¶

func (f *Flag) Bool() bool

Bool gets flag bool value

func (*Flag) Date ¶

func (f *Flag) Date() *time.Time

Date gets flag date value

func (*Flag) Int ¶

func (f *Flag) Int() int

Int gets flag int value

func (*Flag) Set ¶

func (f *Flag) Set(val string) error

Set flag value

func (*Flag) String ¶

func (f *Flag) String() string

String gets string value

type GoogleGroups ¶

type GoogleGroups struct {
	Project     *Flag
	ProjectSlug *Flag
	GroupName   *Flag
	EsIndex     *Flag
	FromDate    *Flag
	DoFetch     *Flag
	DoEnrich    *Flag
	FetchSize   *Flag
	EnrichSize  *Flag
}

GoogleGroups parameter context contains all required parameters to run google groups fetch and enrich

type JiraField ¶

type JiraField struct {
	ID     string `json:"id"`
	Name   string `json:"name"`
	Custom bool   `json:"custom"`
}

JiraField - informatin about fields present in issues

type MemCacheEntry ¶

type MemCacheEntry struct {
	G string    `json:"g"` // cache tag
	B []byte    `json:"b"` // cache data
	T time.Time `json:"t"` // when cached
	E time.Time `json:"e"` // when expires
}

MemCacheEntry - single cache entry

type PLS ¶

type PLS struct {
	Language string `json:"language"`
	Files    int    `json:"files"`
	Blank    int    `json:"blank"`
	Comment  int    `json:"comment"`
	Code     int    `json:"code"`
}

PLS - programming language summary

type PiperMail ¶

type PiperMail struct {
	Origin      *Flag
	Project     *Flag
	ProjectSlug *Flag
	GroupName   *Flag
	EsIndex     *Flag
	FromDate    *Flag
	DoFetch     *Flag
	DoEnrich    *Flag
	FetchSize   *Flag
	EnrichSize  *Flag
}

PiperMail parameter context contains all required parameters to run Piper mail fetch and enrich

type RawPLS ¶

type RawPLS struct {
	Language string `json:"language"`
	Files    string `json:"files"`
	Blank    string `json:"blank"`
	Comment  string `json:"comment"`
	Code     string `json:"code"`
}

RawPLS - programming language summary (all fields as strings)

type Repository ¶

type Repository struct {
	Owner      string
	Repository string
	Project    string
	ESIndex    string
}

Repository dockerhub repository data

Source Files ¶

View all Source files

Directories ¶

Path	Synopsis
affiliation
mocks
bugzilla
mocks
bugzillarest
mocks
build
cmd
dads
db
dockerhub
mocks
googlegroups
mocks
jenkins
pipermail
mocks
test
util

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL

README ¶

dads

Documentation ¶

Index ¶

Constants ¶

Variables ¶

Functions ¶

func AddRedacted ¶

func AffsDataForRoles ¶

func AffsIdentityIDs ¶

func AnonymizeURL ¶

func Base64DecodeCookies ¶

func Base64DecodeHeaders ¶

func Base64EncodeCookies ¶

func Base64EncodeHeaders ¶

func BytesToStringTrunc ¶

func CacheSummary ¶

func CommonFields ¶

func ConfluenceEnrichItemsFunc ¶

func ConnectAffiliationsDB ¶

func CookieToString ¶

func CopyAffsRoleData ¶

func CreateESCache ¶

func DBUploadIdentitiesFunc ¶

func DeepSet ¶

func Dig ¶

func DumpKeys ¶

func DumpPreview ¶

func ESBulkUploadFunc ¶

func ESCacheDelete ¶

func ESCacheDeleteExpired ¶

func ESCacheSet ¶

func EmptyAffsItem ¶

func Enrich ¶

func EnrichComments ¶

func EnrichItem ¶

func EnrichPairProgrammingItem ¶

func EnsurePath ¶

func ExecCommand ¶

func ExecCommandPipe ¶

func ExecDB ¶

func ExecSQL ¶

func ExecTX ¶

func ExecuteAffiliationsAPICall ¶

func FatalOnError ¶

func Fatalf ¶

func FetchRaw ¶

func FilterRedacted ¶

func FindObject ¶

func ForEachESItem ¶

func GerritEnrichItemsFunc ¶

func GetAPIToken ¶

func GetESCache ¶

func GetEnrollments ¶

func GetEnrollmentsBoth ¶

func GetEnrollmentsMulti ¶

func GetEnrollmentsSingle ¶

func GetIdentityUUID ¶

func GetL2Cache ¶

func GetLastOffset ¶

func GetLastUpdate ¶

func GetRedacted ¶

func GetThreadsNum ¶

func GitEnrichItemsFunc ¶

func GitHubEnrichItemsFunc ¶

func GroupsioEnrichItemsFunc ¶

func HandleMapping ¶

func IdentityAffsData ¶

func IdentityAffsDomain ¶

func IndexAt ¶

func InitializeAuth0 ¶

func InterfaceToStringTrunc ¶

func IsOldFormat ¶

func IsValidDomain ¶

func IsValidEmail ¶

func ItemsIdentitiesFunc ¶

func ItemsRefreshIdentitiesFunc ¶

func JSONEscape ¶

func JiraEnrichItemsFunc ¶

func KeysOnly ¶