utils

package
v0.0.0-...-c15b487 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Apr 28, 2024 License: Apache-2.0 Imports: 41 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	DefaultTaskMaxRetries = 3

	QCritical = "critical"
	QDefault  = "default"
	QLow      = "low"
)
View Source 
const (
	Project           = "ThreatMapper"
	ErrorUserNotFound = "user not found"
)
View Source 
const (
	AuditLogs                 = "audit-logs"
	VulnerabilityScan         = "vulnerability-scan"
	VulnerabilityScanStatus   = "vulnerability-scan-status"
	SecretScan                = "secret-scan"
	SecretScanStatus          = "secret-scan-status"
	MalwareScan               = "malware-scan"
	MalwareScanStatus         = "malware-scan-status"
	SbomArtifacts             = "sbom-artifact"
	SbomCVEScan               = "sbom-cve-scan"
	CloudComplianceScan       = "cloud-compliance-scan"
	CloudComplianceScanStatus = "cloud-compliance-scan-status"
	ComplianceScan            = "compliance-scan"
	ComplianceScanStatus      = "compliance-scan-status"
	CloudTrailAlerts          = "cloudtrail-alert"
	CloudResource             = "cloud-resource"
)

kafka topics

View Source 
const (
	CleanUpGraphDBTask                = "clean_up_graph_db"
	CleanUpPostgresqlTask             = "clean_up_postgresql"
	CleanupDiagnosisLogs              = "clean_up_diagnosis_logs"
	RetryFailedScansTask              = "retry_failed_scans"
	RetryFailedUpgradesTask           = "retry_failed_upgrades"
	ScanSBOMTask                      = "tasks_scan_sbom"
	GenerateSBOMTask                  = "tasks_generate_sbom"
	CheckAgentUpgradeTask             = "tasks_check_agent_upgrade"
	SyncRegistryTask                  = "task_sync_registry"
	SyncRegistryPostgresNeo4jTask     = "task_sync_registry_postgres_neo4j"
	TriggerConsoleActionsTask         = "trigger_console_actions"
	ScheduledTasks                    = "scheduled_tasks"
	SecretScanTask                    = "task_secret_scan"
	MalwareScanTask                   = "task_malware_scan"
	ReportGeneratorTask               = "tasks_generate_report"
	ComputeThreatTask                 = "compute_threat"
	SendNotificationTask              = "tasks_send_notification"
	CloudComplianceControlsTask       = "cloud_compliance_controls"
	CachePostureProviders             = "cache_posture_providers"
	ReportCleanUpTask                 = "tasks_cleanup_reports"
	LinkCloudResourceTask             = "link_cloud_resource"
	LinkNodesTask                     = "link_nodes"
	StopSecretScanTask                = "task_stop_secret_scan"
	StopMalwareScanTask               = "task_stop_malware_scan"
	StopVulnerabilityScanTask         = "task_stop_vulnerability_scan"
	UpdateCloudResourceScanStatusTask = "update_cloud_resource_scan_status"
	UpdatePodScanStatusTask           = "update_pod_scan_status"
	BulkDeleteScans                   = "bulk_delete_scans"
	AutoFetchGenerativeAIIntegrations = "auto_fetch_generative_ai_integrations"
	AsynqDeleteAllArchivedTasks       = "asynq_delete_all_archived_tasks"
	RedisRewriteAOF                   = "redis_rewrite_aof"
	DeleteCloudAccounts               = "delete_cloud_accounts"

	UpdateLicenseTask      = "update_license"
	ReportLicenseUsageTask = "report_license_usage"

	ThreatIntelUpdateTask = "threat_intel_update"
)

task names

View Source 
const (
	ScanStatusSuccess       = "COMPLETE"
	ScanStatusStarting      = "STARTING"
	ScanStatusInProgress    = "IN_PROGRESS"
	ScanStatusFailed        = "ERROR"
	ScanStatusCancelPending = "CANCEL_PENDING"
	ScanStatusCancelling    = "CANCELLING"
	ScanStatusCancelled     = "CANCELLED"
	ScanStatusDeletePending = "DELETE_PENDING"
)
View Source 
const (
	NodeTypeCloudProvider     = "CloudProvider"
	NodeTypeCloudRegion       = "CloudRegion"
	NodeTypeKubernetesCluster = "KubernetesCluster"
	NodeTypeContainerImage    = "ContainerImage"
	NodeTypeHost              = "Node"
	NodeTypeContainer         = "Container"
	NodeTypePod               = "Pod"
	NodeTypeProcess           = "Process"
	NodeTypeCloudNode         = "CloudNode"
	NodeTypeCloudResource     = "CloudResource"
	NodeTypeRegistryAccount   = "RegistryAccount"
)

Neo4j Node Labels

View Source 
const (
	MaskGlobal      = "mask_global"
	MaskAllImageTag = "mask_all_image_tag"
	MaskEntity      = "mask_entity"
	MaskImageTag    = "mask_image_tag"
)

mask_global : This is to mask gobally. (same as previous mask_across_hosts_and_images flag) mask_all_image_tag: This is to mask for all tags of an image. mask_entity: This is to mask for an entity other than container/container image. E.g. Host. mask_image_tag: This is to apply mask for an image and tag.

View Source 
const (
	FileServerURLSettingLabel       = "Console File Server URL"
	FileServerURLSettingDescription = "Serve threat intel feeds to agents. If agents are connected using a different URL than Console URL, please change this"
)
View Source 
const (
	FileServerPathAgentBinary = "agent-binary"
)
View Source 
const (
	ReportRetentionTime = 24 * time.Hour
)
View Source 
const (
	ScanRetryFailedStatusMessage = "scan failed to complete, please check agent logs"
)

Variables

View Source 
var (
	ScanTypeDetectedNode = map[Neo4jScanType]string{
		NEO4JVulnerabilityScan:   "Vulnerability",
		NEO4JSecretScan:          "Secret",
		NEO4JMalwareScan:         "Malware",
		NEO4JComplianceScan:      "Compliance",
		NEO4JCloudComplianceScan: "CloudCompliance",
	}
	DetectedNodeScanType = map[string]Neo4jScanType{
		"Vulnerability":   NEO4JVulnerabilityScan,
		"Secret":          NEO4JSecretScan,
		"Malware":         NEO4JMalwareScan,
		"Compliance":      NEO4JComplianceScan,
		"CloudCompliance": NEO4JCloudComplianceScan,
	}
)
View Source 
var (
	ScanIDReplacer = strings.NewReplacer("/", "_", ":", "_", ".", "_")

	SBOMFormatReplacer = strings.NewReplacer("@", "_", ".", "_")
)
View Source 
var (
	AsynqQueues = []string{QCritical, QDefault, QLow}
)
View Source 
var (
	KgoLogger kgo.Logger = kgo.BasicLogger(log.NewIOWriter(zerolog.InfoLevel), kgo.LogLevelInfo, nil)
)

kafka client logger

View Source 
var Tasks = []string{
	CleanUpGraphDBTask,
	CleanUpPostgresqlTask,
	CleanupDiagnosisLogs,
	RetryFailedScansTask,
	RetryFailedUpgradesTask,
	ScanSBOMTask,
	GenerateSBOMTask,
	CheckAgentUpgradeTask,
	SyncRegistryTask,
	TriggerConsoleActionsTask,
	ScheduledTasks,
	SecretScanTask,
	MalwareScanTask,
	ReportGeneratorTask,
	ComputeThreatTask,
	SendNotificationTask,
	CloudComplianceControlsTask,
	CachePostureProviders,
	ReportCleanUpTask,
	LinkCloudResourceTask,
	LinkNodesTask,
	StopSecretScanTask,
	StopMalwareScanTask,
	StopVulnerabilityScanTask,
	UpdateCloudResourceScanStatusTask,
	UpdatePodScanStatusTask,

	UpdateLicenseTask,
	ReportLicenseUsageTask,

	ThreatIntelUpdateTask,
}

Tasks is a list of task names to create topics

View Source 
var Topics = []string{
	VulnerabilityScan, VulnerabilityScanStatus,
	SecretScan, SecretScanStatus,
	MalwareScan, MalwareScanStatus,
	SbomArtifacts, SbomCVEScan,
	CloudComplianceScan, CloudComplianceScanStatus,
	ComplianceScan, ComplianceScanStatus,
	CloudTrailAlerts,
	AuditLogs,
	CloudResource,
}

Functions

func Base64RawDecode 

func Base64RawDecode(s string) (string, error)

func Base64RawEncode 

func Base64RawEncode(s string) string

func BoolArrayToInterfaceArray 

func BoolArrayToInterfaceArray(a []bool) []interface{}

func CheckKafkaConn 

func CheckKafkaConn(kafkaBrokers []string) error

func CreateMissingTopics 

func CreateMissingTopics(
	kafkaBrokers []string,
	topics []string,
	partitions int32,
	replicas int16,
	retentionMS string,
) error

func CritialTaskOpts 

func CritialTaskOpts(opts ...asynq.Option) []asynq.Option

func DefaultTaskOpts 

func DefaultTaskOpts(opts ...asynq.Option) []asynq.Option

func EscapeSpecialCharToUnderscore 

func EscapeSpecialCharToUnderscore(s string) string

func ExecuteCommand 

func ExecuteCommand(commandStr string, envVars map[string]string) (string, error)

func ExtractTarGz 

func ExtractTarGz(gzipStream io.Reader, targetPath string) error

func FileExists 

func FileExists(name string) bool

func FromMap 

func FromMap(bb map[string]interface{}, c interface{})

FromMap Convert map[string]interface{} into structs e.g: 

type Titi struct {
	Tata string `json:"tata"`
}

type Toto struct {
	Foo  string   `json:"foo"`
	Bar  int      `json:"bar"`
	Ta   Titi     `json:"ta"`
	Tas  []Titi   `json:"tas"`
	Tass []string `json:"tass"`
}

m := map[string]interface{}{
	"foo": "toto",
	"bar": 42,
	"ta": map[string]interface{}{"tata": "ok"},
	"tas": []map[string]interface{}{{"tata": "ok2"}, {"tata": "ok1"}},
	"tass": []string{"a"},
}

var t Toto FromMap(m, &t)

func GenerateHashFromString 

func GenerateHashFromString(s string) string

func GenerateRandomNumber 

func GenerateRandomNumber(numberOfDigits int) (int64, error)

func GetBoolValueFromInterfaceMap 

func GetBoolValueFromInterfaceMap(claims map[string]interface{}, key string) (bool, error)

func GetCurrentDatetime 

func GetCurrentDatetime() time.Time

func GetCustomerNamespace 

func GetCustomerNamespace(s string) (string, error)

func GetDatetimeNow 

func GetDatetimeNow() string

func GetEmailDomain 

func GetEmailDomain(email string) (string, error)

func GetEnvOrDefault 

func GetEnvOrDefault(envVar string, defaultValue string) string

func GetEnvOrDefaultInt 

func GetEnvOrDefaultInt(envVar string, defaultValue int) int

func GetErrorRedirectURL 

func GetErrorRedirectURL(consoleURL, errorMessage string) string

func GetHTTPClient 

func GetHTTPClient() *http.Client

func GetInfoRedirectURL 

func GetInfoRedirectURL(urlPath, message string) string

func GetInsecureHTTPClient 

func GetInsecureHTTPClient() *http.Client

func GetInt64ValueFromInterfaceMap 

func GetInt64ValueFromInterfaceMap(claims map[string]interface{}, key string) (int64, error)

func GetRegistryID 

func GetRegistryID(registryType, ns string, pgID int32) string

func GetScheduledJobHash 

func GetScheduledJobHash(schedule postgresqlDb.Scheduler) string

func GetStringValueFromInterfaceMap 

func GetStringValueFromInterfaceMap(claims map[string]interface{}, key string) (string, error)

func GetTimestamp 

func GetTimestamp() int64

func InSlice 

func InSlice[T comparable](e T, s []T) bool

func InterfaceToInt 

func InterfaceToInt(a interface{}) (int64, error)

func IsJWTExpired 

func IsJWTExpired(token string) bool

func IsUUIDValid 

func IsUUIDValid(uuidStr string) bool

func LowTaskOpts 

func LowTaskOpts(opts ...asynq.Option) []asynq.Option

func MapKeys 

func MapKeys(input map[string]string) []int32

func NewHTTPClient 

func NewHTTPClient() (*http.Client, error)

func NewUUID 

func NewUUID() uuid.UUID

func NewUUIDString 

func NewUUIDString() string

func PrintableTimeStamp 

func PrintableTimeStamp(timestamp interface{}) string

UNIX timestamp to common readable format

func RandomString 

func RandomString(nByte int) (string, error)

func RecursiveZip 

func RecursiveZip(pathsToZip []string, excludePathPrefixes []string, destinationPath string) error

func RemoveURLPath 

func RemoveURLPath(inURL string) (string, error)

func ResourceTypeToNeo4jLabel 

func ResourceTypeToNeo4jLabel(t CloudProvider) string

func SHA256sum 

func SHA256sum(data []byte) string

func SetCookie 

func SetCookie(w http.ResponseWriter, r *http.Request, name, value, path string)

func SplitFullName 

func SplitFullName(name string) (string, string)

func StartKafkaProducer 

func StartKafkaProducer(
	ctx context.Context,
	brokers []string,
	ingestChan chan *kgo.Record,
)

func StringArrayToInterfaceArray 

func StringArrayToInterfaceArray(a []string) []interface{}

func StripAnsi 

func StripAnsi(str []byte) []byte

StripAnsi remove ansi color from log lines

func StripAnsiStr 

func StripAnsiStr(str string) string

StripAnsiStr remove ansi color from log lines

func StructToMap 

func StructToMap[T any](c T) map[string]interface{}

func TaskOptions 

func TaskOptions(queue string, opts ...asynq.Option) []asynq.Option

func TasksMaxRetries 

func TasksMaxRetries() asynq.Option

func ToMap 

func ToMap[T any](c T) map[string]interface{}

TODO: check if StructToMap can replace ToMap

func ToSnakeCase 

func ToSnakeCase(str string) string

func TopicWithNamespace 

func TopicWithNamespace(topic, ns string) string

func TopicsWithNamespace 

func TopicsWithNamespace(ns string) []string

func URLDecode 

func URLDecode(s string) (string, error)

func URLEncode 

func URLEncode(s string) string

func UUIDFromString 

func UUIDFromString(uuidStr string) (uuid.UUID, error)

func UploadFile 

func UploadFile(url string, fileName string) ([]byte, int, error)

func ValidateEmail 

func ValidateEmail(email string) bool

func WaitServiceTCPConn 

func WaitServiceTCPConn(host string, port string, timeout time.Duration) error

Types

type AdvancedReportFilters 

type AdvancedReportFilters struct {
	Masked                []bool   `json:"masked,omitempty"`
	ScanStatus            []string `json:"scan_status,omitempty"`
	PodName               []string `json:"pod_name,omitempty"`
	ContainerName         []string `json:"container_name,omitempty"`
	ImageName             []string `json:"image_name,omitempty"`
	HostName              []string `json:"host_name,omitempty"`
	AccountID             []string `json:"node_id,omitempty"`
	KubernetesClusterName []string `json:"kubernetes_cluster_name,omitempty"`
}

type AgentID 

type AgentID struct {
	Name string `json:"name"`
	Type string `json:"type"`
}

type AutoFetchGenerativeAIIntegrationsParameters 

type AutoFetchGenerativeAIIntegrationsParameters struct {
	CloudProvider string `json:"cloud_provider"`
	UserID        int64  `json:"user_id"`
}

type CloudProvider 

type CloudProvider int
const (
	AWS CloudProvider = iota
	GCP
	Azure
	DO
	AWSOrg
	GCPOrg
)

func StringToCloudProvider 

func StringToCloudProvider(s string) CloudProvider

type MalwareScanParameters 

type MalwareScanParameters struct {
	ImageName             string `json:"image_name"`
	ImageID               string `json:"image_id"`
	ScanID                string `json:"scan_id" required:"true"`
	KubernetesClusterName string `json:"kubernetes_cluster_name"`
	HostName              string `json:"host_name"`
	NodeID                string `json:"node_id"`
	NodeType              string `json:"node_type"`
	ScanType              string `json:"scan_type"`
	ContainerName         string `json:"container_name"`
	Mode                  string `json:"mode,omitempty"`
	RegistryID            string `json:"registry_id,omitempty"`
}

type MinioError 

type MinioError struct {
	XMLName    xml.Name `xml:"Error"`
	Text       string   `xml:",chardata"`
	Code       string   `xml:"Code"`
	Message    string   `xml:"Message"`
	Key        string   `xml:"Key"`
	BucketName string   `xml:"BucketName"`
	Resource   string   `xml:"Resource"`
	RequestID  string   `xml:"RequestID"`
	HostID     string   `xml:"HostID"`
}

type Neo4jScanType 

type Neo4jScanType string
const (
	NEO4JSecretScan          Neo4jScanType = "SecretScan"
	NEO4JVulnerabilityScan   Neo4jScanType = "VulnerabilityScan"
	NEO4JMalwareScan         Neo4jScanType = "MalwareScan"
	NEO4JComplianceScan      Neo4jScanType = "ComplianceScan"
	NEO4JCloudComplianceScan Neo4jScanType = "CloudComplianceScan"
)

func StringToNeo4jScanType 

func StringToNeo4jScanType(s string) Neo4jScanType

type RegistrySyncParams 

type RegistrySyncParams struct {
	PgID int32 `json:"pg_id"`
}

type ReportFilters 

type ReportFilters struct {
	ScanID                string                `json:"scan_id"`
	ScanType              string                `json:"scan_type" validate:"required" required:"true" enum:"vulnerability,secret,malware,compliance,cloud_compliance"`
	NodeType              string                `json:"node_type" validate:"required" required:"true" enum:"host,container,container_image,linux,cluster,aws,gcp,azure"`
	SeverityOrCheckType   []string              `json:"severity_or_check_type" enum:"critical,high,medium,low,cis,gdpr,nist,hipaa,pci,soc_2"`
	IncludeDeadNode       bool                  `json:"include_dead_nodes"`
	MostExploitableReport bool                  `json:"most_exploitable_report"`
	AdvancedReportFilters AdvancedReportFilters `json:"advanced_report_filters,omitempty"`
}

func (ReportFilters) String 

func (r ReportFilters) String() string

type ReportOptions 

type ReportOptions struct {
	// SBOMFormat Applicable if ReportType is sbom
	SBOMFormat string `` /* 180-byte string literal not displayed */
}

type ReportParams 

type ReportParams struct {
	ReportID      string        `json:"report_id"`
	ReportType    string        `json:"report_type"`
	FromTimestamp time.Time     `json:"from_timestamp"`
	ToTimestamp   time.Time     `json:"to_timestamp"`
	Filters       ReportFilters `json:"filters"`
	Options       ReportOptions `json:"options,omitempty"`
}

type ReportType 

type ReportType string
const (
	ReportXLSX ReportType = "xlsx"
	ReportPDF  ReportType = "pdf"
	ReportSBOM ReportType = "sbom"
)

type SbomBody 

type SbomBody struct {
	SBOM string `json:"sbom" required:"true"`
}

type SbomParameters 

type SbomParameters struct {
	ImageName             string `json:"image_name"`
	ImageID               string `json:"image_id"`
	ScanID                string `json:"scan_id" required:"true"`
	KubernetesClusterName string `json:"kubernetes_cluster_name"`
	HostName              string `json:"host_name"`
	NodeID                string `json:"node_id"`
	NodeType              string `json:"node_type"`
	ScanType              string `json:"scan_type"`
	ContainerName         string `json:"container_name"`
	SBOMFilePath          string `json:"sbom_file_path"`
	Mode                  string `json:"mode,omitempty"`
	RegistryID            string `json:"registry_id,omitempty"`
	SkipScan              bool   `json:"skip_scan,omitempty"`
}

type ScanSbomRequest 

type ScanSbomRequest struct {
	SbomParameters
	SbomBody
}

type SecretScanParameters 

type SecretScanParameters struct {
	ImageName             string `json:"image_name"`
	ImageID               string `json:"image_id"`
	ScanID                string `json:"scan_id" required:"true"`
	KubernetesClusterName string `json:"kubernetes_cluster_name"`
	HostName              string `json:"host_name"`
	NodeID                string `json:"node_id"`
	NodeType              string `json:"node_type"`
	ScanType              string `json:"scan_type"`
	ContainerName         string `json:"container_name"`
	Mode                  string `json:"mode,omitempty"`
	RegistryID            string `json:"registry_id,omitempty"`
}

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.