Why Go
- Case Studies
  
  Common problems companies solve with Go
- Use Cases
  
  Stories about how and why companies use Go
- Security Policy
  
  How Go can help keep you secure by default
Learn
Docs
- Effective Go
  
  Tips for writing clear, performant, and idiomatic Go code
- Go User Manual
  
  A complete introduction to building software with Go
- Standard library
  
  Reference documentation for Go's standard library
- Release Notes
  
  Learn what's new in each Go release
Packages
Community
- Recorded Talks
  
  Videos from prior events
- Meetups
  
  Meet other local Go developers
- Conferences
  
  Learn and network with Go developers from around the world
- Go blog
  
  The Go project's official blog.
- Go project
  
  Get help and stay informed from Go
- Get connected

ds

package module

Go to main page

Versions in this module

v1

v1.5.32

Apr 27, 2023

v1.5.31

Apr 19, 2023

v1.5.30

Apr 18, 2023

v1.5.29

Mar 10, 2023

v1.5.28

Mar 9, 2023

v1.5.27

Mar 3, 2023

v1.5.26

Feb 28, 2023

v1.5.25

Feb 3, 2023

v1.5.24

Jan 31, 2023

v1.5.23

Jan 26, 2023

v1.5.22

Jan 26, 2023

v1.5.21

Dec 22, 2022

v1.5.20

Nov 5, 2022

v1.5.19

Nov 4, 2022

v1.5.18

Oct 31, 2022

v1.5.17

Oct 19, 2022

v1.5.16

Sep 16, 2022

v1.5.15

Sep 14, 2022

v1.5.14

Sep 8, 2022

v1.5.13

Aug 30, 2022

v1.5.12

Aug 9, 2022

v1.5.11

Aug 5, 2022

v1.5.10

Jul 29, 2022

v1.5.9

Jul 27, 2022

v1.5.8

Jul 27, 2022

v1.5.7

Jul 26, 2022

v1.5.6

Jul 22, 2022

v1.5.5

Jul 21, 2022

v1.5.4

Jul 21, 2022

v1.5.3

Jun 14, 2022

v1.5.2

Jun 13, 2022

v1.5.1

Jun 12, 2022

v1.5.0

Jun 11, 2022

v1.4.4

Mar 31, 2022

v1.4.3

Mar 25, 2022

v1.4.2

Mar 11, 2022

v1.4.1

Feb 2, 2022

v1.4.0

Jan 31, 2022

v1.3.0

Jan 28, 2022

v1.2.3

Nov 15, 2022

Changes in this version

+ func AddLogger(logger *logger.Logger, connector, status string, ...)

+ func DedupContributors(inContributors []insights.Contributor) (outContributors []insights.Contributor)

+ func IsBotIdentity(name, username, email, datasource string, botNameR string, botUsernameR string, ...) bool

+ func SetLogLoggerError(logLoggerError bool)

+ func SetSyncMode(sync, consoleAfterES bool)

+ func StripURL(urlStr string) string

type Ctx

+ NoIncremental bool

v1.2.2

Jan 25, 2022

v1.2.1

Jan 21, 2022

v1.2.0

Jan 19, 2022

v1.1.3

Dec 21, 2021

v1.1.2

Dec 7, 2021

v1.1.1

Nov 19, 2021

v1.1.0

Nov 2, 2021

v1.0.0

Oct 6, 2021

Changes in this version

+ const CacheCleanupProb

+ const DefaultPackSize

+ const DefaultRateLimitHeader

+ const DefaultRateLimitResetHeader

+ const KeywordMaxlength

+ const MBoxDropXFields

+ const MaxBodyLength

+ const MaxPayloadPrintfLen

+ const MissingName

+ const RedactedEmail

+ var AnonymizeURLPattern = regexp.MustCompile(`(^.*)(://)(.*@)(.*$)`)

+ var CloseAddrRE = regexp.MustCompile(`\s+>`)

+ var DefaultDateFrom = time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC)

+ var DefaultDateTo = time.Date(2100, 1, 1, 0, 0, 0, 0, time.UTC)

+ var EmailRegex = regexp.MustCompile(...)

+ var EmailReplacer = strings.NewReplacer(" at ", "@", " AT ", "@", " At ", "@", " dot ", ".", " DOT ", ".", " Dot ", ...)

+ var GRedactedMtx *sync.RWMutex

+ var GRedactedStrings map[string]struct

+ var LowerDayNames = map[string]struct

+ var LowerFullMonthNames = map[string]string

+ var LowerMonthNames = map[string]string

+ var MBoxMsgSeparator = map[string][]byte

+ var MT = false

+ var MaxMessageBodyLength = map[string]int

+ var MaxMessageProperties = map[string]int

+ var MessageDateField = map[string]string

+ var MessageIDField = map[string]string

+ var MessageReceivedField = map[string]string

+ var MsgLineSeparator = map[string][]byte

+ var OpenAddrRE = regexp.MustCompile(`<\s+`)

+ var RawFields = []string

+ var SpacesRE = regexp.MustCompile(`\s+`)

+ var TZOffsetRE = regexp.MustCompile(`^(\d{3})(\s+.*$|$)`)

+ var WhiteSpace = regexp.MustCompile(`\s+`)

+ func AddRedacted(newRedacted string, useMutex bool)

+ func AnonymizeURL(url string) string

+ func AsJSON(data interface{}) string

+ func Base64DecodeCookies(enc []byte) (cookies []string, err error)

+ func Base64DecodeHeaders(enc []byte) (headers map[string][]string, err error)

+ func Base64EncodeCookies(cookies []string) (enc []byte)

+ func Base64EncodeHeaders(headers map[string][]string) (enc []byte)

+ func BytesToStringTrunc(data []byte, maxLen int, addLenInfo bool) (str string)

+ func ConvertTimeToFloat(t time.Time) float64

+ func CookieToString(c *http.Cookie) (s string)

+ func CreateESCache(ctx *Ctx)

+ func DeepSet(m interface{}, ks []string, v interface{}, create bool) (err error)

+ func Dig(iface interface{}, keys []string, fatal, silent bool) (v interface{}, ok bool)

+ func DumpKeys(i interface{}) string

+ func DumpPreview(i interface{}, l int) string

+ func ESCacheDelete(ctx *Ctx, key string)

+ func ESCacheDeleteExpired(ctx *Ctx)

+ func ESCacheSet(ctx *Ctx, key string, entry *ESCacheEntry)

+ func EnsurePath(path string, noLastDir bool) (string, error)

+ func ExecCommand(ctx *Ctx, cmdAndArgs []string, cwd string, env map[string]string) (sout, serr string, err error)

+ func ExecCommandPipe(ctx *Ctx, cmdAndArgs []string, cwd string, env map[string]string) (stdOutPipe io.ReadCloser, cmd *exec.Cmd, err error)

+ func FatalOnError(err error) string

+ func Fatalf(f string, a ...interface{})

+ func FilterRedacted(str string) string

+ func FlagPassed(ctx *Ctx, name string) bool

+ func GetDaysBetweenDates(t1 time.Time, t2 time.Time) float64

+ func GetESCache(ctx *Ctx, k string) (b []byte, tg string, expires time.Time, ok bool)

+ func GetL2Cache(ctx *Ctx, k string) (b []byte, ok bool)

+ func GetLastUpdate(ctx *Ctx, key string) (lastUpdate *time.Time)

+ func GetOldestDate(t1 *time.Time, t2 *time.Time) *time.Time

+ func GetRedacted() (str string)

+ func GetThreadsNum(ctx *Ctx) int

+ func IndexAt(s, sep string, n int) int

+ func InterfaceToStringTrunc(iface interface{}, maxLen int, addLenInfo bool) (str string)

+ func IsValidDomain(domain string) (valid bool)

+ func IsValidEmail(email string, validateDomain, guess bool) (valid bool, newEmail string)

+ func JSONEscape(str string) string

+ func KeysOnly(i interface{}) (o map[string]interface{})

+ func MatchGroups(re *regexp.Regexp, arg string) (result map[string]string)

+ func MatchGroupsArray(re *regexp.Regexp, arg string) (result map[string][]string)

+ func MaybeESCacheCleanup(ctx *Ctx)

+ func MaybeMemCacheCleanup(ctx *Ctx)

+ func MemCacheDeleteExpired(ctx *Ctx)

+ func NoSSLVerify()

+ func ParseAddresses(ctx *Ctx, addrs string, maxAddrs int) (emails []*mail.Address, ok bool)

+ func ParseDateWithTz(indt string) (dt, dtInTz time.Time, off float64, valid bool)

+ func ParseMBoxMsg(ctx *Ctx, groupName string, msg []byte, dsType string) (item map[string]interface{}, valid, warn bool)

+ func PartitionString(s string, sep string) [3]string

+ func PeriodParse(perStr string) (dur time.Duration, ok bool)

+ func PostprocessNameUsername(name, username, email string) (outName, outUsername string)

+ func PrettyPrint(data interface{}) string

+ func PrettyPrintJSON(jsonBytes []byte) []byte

+ func PreviewOnly(i interface{}, l int) (o interface{})

+ func Printf(format string, args ...interface{})

+ func PrintfNoRedacted(format string, args ...interface{})

+ func RedactEmail(in, suff string, forceSuff bool) string

+ func Request(ctx *Ctx, url, method string, headers map[string]string, payload []byte, ...) (result interface{}, status int, outCookies []string, ...)

+ func RequestNoRetry(ctx *Ctx, url, method string, headers map[string]string, payload []byte, ...) (result interface{}, status int, isJSON bool, outCookies []string, ...)

+ func ResetThreadsNum(ctx *Ctx)

+ func ResetUUIDCache()

+ func SetESCache(ctx *Ctx, k, tg string, b []byte, expires time.Duration)

+ func SetL2Cache(ctx *Ctx, k, tg string, b []byte, expires time.Duration)

+ func SetLastUpdate(ctx *Ctx, key string, when time.Time)

+ func SetMT()

+ func StringToBool(v string) bool

+ func StringToCookie(s string) (c *http.Cookie)

+ func StringTrunc(data string, maxLen int, addLenInfo bool) (str string)

+ func TimeParseAny(dtStr string) (time.Time, error)

+ func TimeParseES(dtStr string) (time.Time, error)

+ func TimeParseInterfaceString(date interface{}) (dt time.Time, err error)

+ func ToESDate(dt time.Time) string

+ func ToYMDHMDate(dt time.Time) string

+ func ToYMDHMSDate(dt time.Time) string

+ func ToYMDTHMSZDate(dt time.Time) string

+ func UUIDAffs(ctx *Ctx, args ...string) (h string)

+ func UUIDNonEmpty(ctx *Ctx, args ...string) (h string)

+ func UniqueStringArray(ary []interface{}) []interface

+ type Ctx struct

+ Categories map[string]struct{}

+ DS string

+ DSEnv string

+ DSFlag string

+ DateFrom *time.Time

+ DateTo *time.Time

+ Debug int

+ DryRun bool

+ ESURL string

+ NCPUs int

+ NCPUsScale float64

+ NoCache bool

+ PackSize int

+ Project string

+ ProjectFilter bool

+ Retry int

+ ST bool

+ Tags []string

+ func (ctx *Ctx) BoolEnv(k string) bool

+ func (ctx *Ctx) BoolEnvSet(k string) (bool, bool)

+ func (ctx *Ctx) Env(k string) string

+ func (ctx *Ctx) EnvSet(k string) bool

+ func (ctx *Ctx) Init()

+ func (ctx *Ctx) InitEnv(dsName string)

+ func (ctx *Ctx) Print()

+ func (ctx Ctx) Info() string

+ type DateCacheEntry struct

+ Dt time.Time

+ DtInTz time.Time

+ TzOff float64

+ Valid bool

+ type ESCacheEntry struct

+ B []byte

+ E time.Time

+ G string

+ K string

+ T time.Time

+ func ESCacheGet(ctx *Ctx, key string) (entry *ESCacheEntry, ok bool)

+ type MemCacheEntry struct

+ B []byte

+ E time.Time

+ G string

+ T time.Time