googlegroups

package
v0.0.0-...-b175f30 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jun 23, 2022 License: Apache-2.0 Imports: 24 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	// GoogleGroups ...
	GoogleGroups = "googlegroups"

	// Unknown ...
	Unknown = "Unknown"
	// CredentialsSSMParamName from ssm
	CredentialsSSMParamName = "insights_googlegroups_credentials"
	// TokenSSMParamName from ssm
	TokenSSMParamName = "insights_googlegroups_token"
	// MaxNumberOfMessages from gmail
	MaxNumberOfMessages = 10000000
	// MaxConcurrentRequests ...
	MaxConcurrentRequests = 10000
)

Variables

View Source 
var (
	// GoogleGroupRichMapping ...
	GoogleGroupRichMapping = []byte(`{"mappings":{"dynamic_templates":[{"notanalyzed":{"match":"*","match_mapping_type":"string","mapping":{"type":"keyword"}}},{"int_to_float":{"match":"*","match_mapping_type":"long","mapping":{"type":"float"}}},{"formatdate":{"match":"*","match_mapping_type":"date","mapping":{"format":"strict_date_optional_time||epoch_millis","type":"date"}}}]}}`)
	// GoogleGroupRawMapping ...
	GoogleGroupRawMapping = []byte(`{"mappings":{"dynamic":true,"properties":{"metadata__updated_on":{"type":"date"},"data":{"properties":{"body":{"dynamic":false,"properties":{}}}}}}}`)

	// DefaultDateTime ...
	DefaultDateTime = time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC)
)

Functions

This section is empty.

Types

type AffiliationClient 

type AffiliationClient interface {
	GetIdentityByUser(key string, value string) (*affiliation.AffIdentity, error)
	AddIdentity(identity *affiliation.Identity) bool
	GetOrganizations(uuid string, projectSlug string) *[]affiliation.Enrollment
}

AffiliationClient manages user identity

type AuthClientProvider 

type AuthClientProvider interface {
	GetToken(env string) (string, error)
}

AuthClientProvider interacts with auth0 server

type EnrichedMessage 

type EnrichedMessage struct {
	From                 string    `json:"from"`
	Date                 time.Time `json:"date"`
	To                   []string  `json:"to"`
	MessageID            string    `json:"message_id"`
	InReplyTo            string    `json:"in_reply_to"`
	References           string    `json:"references"`
	Subject              string    `json:"subject"`
	Topic                string    `json:"topic"`
	MessageBody          string    `json:"message_body"`
	TopicID              string    `json:"topic_id"`
	BackendVersion       string    `json:"backend_version"`
	UUID                 string    `json:"uuid"`
	Origin               string    `json:"origin"`
	MetadataUpdatedOn    time.Time `json:"metadata__updated_on"`
	BackendName          string    `json:"backend_name"`
	MetadataTimestamp    time.Time `json:"metadata__timestamp"`
	MetadataEnrichedOn   time.Time `json:"metadata__enriched_on"`
	ProjectSlug          string    `json:"project_slug"`
	GroupName            string    `json:"group_name"`
	Project              string    `json:"project"`
	Root                 bool      `json:"root"`
	FromBot              bool      `json:"from_bot"`
	ChangedAt            time.Time `json:"changed_at"`
	AuthorName           string    `json:"author_name"`
	AuthorID             string    `json:"author_id"`
	AuthorUUID           string    `json:"author_uuid"`
	AuthorOrgName        string    `json:"author_org_name"`
	AuthorUserName       string    `json:"author_user_name"`
	AuthorBot            bool      `json:"author_bot"`
	AuthorMultiOrgNames  []string  `json:"author_multi_org_names"`
	MboxAuthorDomain     string    `json:"mbox_author_domain"`
	IsGoogleGroupMessage int       `json:"is_google_group_message"`
	Timezone             int       `json:"timezone"`
	ViaCommunityGroup    bool      `json:"via_community_group"`
}

EnrichedMessage ...

type Enricher 

type Enricher struct {
	DSName                string // Datasource will be used as key for ES
	ElasticSearchProvider *elastic.ClientProvider
	// contains filtered or unexported fields
}

Enricher contains google groups datasource enrich logic

func NewEnricher 

func NewEnricher(esClientProvider *elastic.ClientProvider, affiliationsClientProvider *affiliation.Affiliation) *Enricher

NewEnricher initiates a new Enricher

func (*Enricher) EnrichMessage 

func (e *Enricher) EnrichMessage(rawMessage *RawMessage, now time.Time) (*EnrichedMessage, error)

EnrichMessage enriches raw message

func (*Enricher) Find 

func (e *Enricher) Find(slice []string, val string) bool

Find takes a slice and looks for an element in it. If found it will return it's true, otherwise it will return false.

func (*Enricher) GetEmailAddress 

func (e *Enricher) GetEmailAddress(rawMailString string) (mail *string)

GetEmailAddress ...

func (*Enricher) GetEmailDomain 

func (e *Enricher) GetEmailDomain(email string) string

GetEmailDomain ...

func (*Enricher) GetEmailUsername 

func (e *Enricher) GetEmailUsername(email string) string

GetEmailUsername ...

func (*Enricher) GetUserName 

func (e *Enricher) GetUserName(rawMailString string) (username string)

GetUserName ...

func (*Enricher) IsValidEmail 

func (e *Enricher) IsValidEmail(rawMailString string) bool

IsValidEmail validates email string

func (*Enricher) RemoveSpecialCharactersFromString 

func (e *Enricher) RemoveSpecialCharactersFromString(s string) (val *string)

RemoveSpecialCharactersFromString ...

func (*Enricher) SanitizeEmails 

func (e *Enricher) SanitizeEmails(emails []string) string

SanitizeEmails returns the first well formed email address for a given user it filters out norely emails and emails with ellipsis

type Fetcher 

type Fetcher struct {
	DSName                string
	HTTPClientProvider    *http.ClientProvider
	ElasticSearchProvider *elastic.ClientProvider
	BackendVersion        string
	DateFrom              time.Time
	GroupName             string
	ProjectSlug           string
	Project               string
}

Fetcher contains GoogleGroups datasource fetch logic

func NewFetcher 

func NewFetcher(groupName, projectSlug, project string, httpClientProvider *http.ClientProvider, esClientProvider *elastic.ClientProvider) *Fetcher

NewFetcher initiates a new GoogleGroups fetcher

func (*Fetcher) Fetch 

func (f *Fetcher) Fetch(fromDate, now *time.Time) ([]*RawMessage, error)

Fetch ...

type GoogleGroupMessage 

type GoogleGroupMessage struct {
	ID      string `json:"id"`
	Author  string `json:"author"`
	Date    string `json:"date"`
	File    string `json:"file"`
	Message string `json:"message"`
}

GoogleGroupMessage ...

type GoogleGroupMessageThread 

type GoogleGroupMessageThread struct {
	Topic    string                `json:"topic"`
	ID       string                `json:"id"`
	Messages []*GoogleGroupMessage `json:"messages"`
}

GoogleGroupMessageThread ...

type GoogleGroupMessages 

type GoogleGroupMessages struct {
	Messages []*GoogleGroupMessageThread
}

GoogleGroupMessages ...

type HeadersData 

type HeadersData struct {
	// Date is the date the message was originally sent
	Date string
	// MessageID is the message id
	MessageID string
	// InReplyTo is who the email was sent to. This can contain multiple
	// addresses if the email was forwarded.
	InReplyTo string
	// References
	References string
	// Sender is the entity that originally created and sent the message
	Sender string
	// From is the name - email address combo of the email author
	From string
	// Subject is the subject of the email
	Subject string
	// To is the email recipient.
	To []string
	// DeliveredTo is to whom the email was sent to. This can contain multiple
	// addresses if the email was forwarded.
	DeliveredTo []string
	MailingList string
}

HeadersData struct

type HitSource 

type HitSource struct {
	ID        string    `json:"id"`
	ChangedAt time.Time `json:"changed_at"`
}

HitSource is the document _source data

type Hits 

type Hits struct {
	Hits []NestedHits `json:"hits"`
}

Hits result

type Manager 

type Manager struct {
	Slug                   string
	GroupName              string
	SHConnString           string
	FetcherBackendVersion  string
	EnricherBackendVersion string
	Fetch                  bool
	Enrich                 bool
	ESUrl                  string
	ESUsername             string
	ESPassword             string
	ESIndex                string
	FromDate               *time.Time
	HTTPTimeout            time.Duration
	Project                string
	FetchSize              int
	EnrichSize             int
	AffBaseURL             string
	ESCacheURL             string
	ESCacheUsername        string
	ESCachePassword        string
	AuthGrantType          string
	AuthClientID           string
	AuthClientSecret       string
	AuthAudience           string
	Auth0URL               string
	Environment            string
	WebHookURL             string
	// contains filtered or unexported fields
}

Manager describes google groups manager

func NewManager 

func NewManager(slug, groupName, shConnStr, fetcherBackendVersion, enricherBackendVersion string, fetch bool, enrich bool, eSUrl string, esUser string, esPassword string, esIndex string, fromDate *time.Time, project string, fetchSize int, enrichSize int, affBaseURL, esCacheURL, esCacheUsername, esCachePassword, authGrantType, authClientID, authClientSecret, authAudience, authURL, env string) (*Manager, error)

NewManager initiates google groups manager instance

func (*Manager) AddTask 

func (m *Manager) AddTask(task func())

AddTask adds task to worker pool

func (*Manager) GetTotalQueuedTask 

func (m *Manager) GetTotalQueuedTask() int

GetTotalQueuedTask get total number of queued tasks

func (*Manager) Sync 

func (m *Manager) Sync() error

Sync runs google groups fetch and enrich according to passed parameters

type NHits 

type NHits struct {
	Hits []NestedRawHits `json:"hits"`
}

NHits result

type NestedHits 

type NestedHits struct {
	ID     string    `json:"_id"`
	Source HitSource `json:"_source"`
}

NestedHits is the actual hit data

type NestedRawHits 

type NestedRawHits struct {
	ID     string     `json:"_id"`
	Source RawMessage `json:"_source"`
}

NestedRawHits is the actual hit data

type RawHits 

type RawHits struct {
	Hits NHits `json:"hits"`
}

RawHits result

type RawMessage 

type RawMessage struct {
	From              string    `json:"from"`
	Date              time.Time `json:"date"`
	To                []string  `json:"to"`
	MessageID         string    `json:"message_id"`
	InReplyTo         string    `json:"in_reply_to"`
	References        string    `json:"references"`
	Subject           string    `json:"subject"`
	MessageBody       string    `json:"message_body"`
	TopicID           string    `json:"topic_id"`
	Topic             string    `json:"topic"`
	BackendVersion    string    `json:"backend_version"`
	UUID              string    `json:"uuid"`
	Origin            string    `json:"origin"`
	MetadataUpdatedOn time.Time `json:"metadata__updated_on"`
	BackendName       string    `json:"backend_name"`
	MetadataTimestamp time.Time `json:"metadata__timestamp"`
	ProjectSlug       string    `json:"project_slug"`
	GroupName         string    `json:"group_name"`
	Project           string    `json:"project"`
	ChangedAt         time.Time `json:"changed_at"`
	Timezone          int       `json:"timezone"`
}

RawMessage represents GoogleGroups raw message

type TopHits 

type TopHits struct {
	Hits Hits `json:"hits"`
}

TopHits result

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.