helpers

Documentation

Index

• Variables
• func BufferFromFile(inputfile *os.File) (*bufio.Reader, error)
• func BuildThreatDB(arguments map[string]any, logger zerolog.Logger) error
• func CheckIPinTI(ip string, isDataCenter bool, db *sql.DB) (string, string, string, bool, error)
• func CloseChannelWhenDone(c chan []string, wg *lbtypes.WaitGroupCount)
• func CombineOutputs(arguments map[string]any, logger zerolog.Logger) error
• func CopyFile(src, dst string) error
• func CreateOutput(outputFile string) (*os.File, error)
• func DoDNSEnrichment(ipaddress string) (string, string)
• func DoDomainWhoisenrichment(domain string) []string
• func DoIDBEnrichment(ipaddress string) []string
• func DoIPWhoisEnrichment(ipaddress string) []string
• func DoesFileExist(filename string) bool
• func DownloadAuthenticatedFile(logger zerolog.Logger, url string, filepath string, key string, user string, ...) (err error)
• func DownloadFile(logger zerolog.Logger, url string, filepath string, key string) (err error)
• func ExtractTarGz(gzipStream io.Reader, logger zerolog.Logger, dir string) error
• func FileToSlice(filename string, logger zerolog.Logger) []string
• func FindOrGetDBs(arguments map[string]any, logger zerolog.Logger, apikey string) error
• func FindTargetIndexInSlice(headers []string, targetCol string) int
• func GetFeedIDIfExist(feed_name string, db *sql.DB) int
• func GetHeaders(tempArgs map[string]any, headers []string) []string
• func GetNewPW(logger zerolog.Logger, inputFile string, outputFile string) (*csv.Reader, *csv.Writer, *os.File, *os.File, error)
• func Increment(ip net.IP)
• func IngestFile(inputFile string, categories string, feedid int, db *sql.DB, ...) error
• func IngestIPNetLists(url string, name string, file string, listtype string, category string, ...)
• func InsertCategory(category string, db *sql.DB) error
• func InsertFeed(feed_name string, feed_url string, db *sql.DB) (error, int)
• func IntSlicetoStringSlice(s []int) []string
• func IsASN(s string) bool
• func IsPrivateIP(ip net.IP, ipstring string) bool
• func ListenOnWriteChannel(c chan []string, w *csv.Writer, logger zerolog.Logger, outputF *os.File, ...)
• func LookupIPRecords(ip string) []string
• func OpenDBConnection(logger zerolog.Logger) (*sql.DB, error)
• func OpenInput(inputFile string) (*os.File, error)
• func ProcessRecords(logger zerolog.Logger, records [][]string, asnDB maxminddb.Reader, ...)
• func RegexFirstPublicIPFromString(input string) (string, bool)
• func RemoveSpace(s string) string
• func ScannerFromFile(reader io.Reader) (*bufio.Scanner, error)
• func SetAPIUrls(arguments map[string]any, logger zerolog.Logger) (error, string)
• func SetupLogger() zerolog.Logger
• func SetupPrivateNetworks() error
• func SummarizeThreatDB(logger zerolog.Logger)
• func SummarizeThreatFeeds(logger zerolog.Logger)
• func UpdateDCList(logger zerolog.Logger)
• func UpdateVPNList(logger zerolog.Logger)
• func Whois(domain string, servers ...string) (result string, err error)
• type Client
  • func NewClient() *Client
  • func (c *Client) Whois(domain string, servers ...string) (result string, err error)
• type Feed
• type Feeds
• type IPNetGenerator
  • func NewFromIPNet(ipNet *net.IPNet) *IPNetGenerator
  • func NewIPNetGenerator(cidr string) (*IPNetGenerator, error)
  • func (g *IPNetGenerator) Next() net.IP
• type IPWhoisResult
  • func ParseIPWhoisLookup(data string) (IPWhoisResult, error)

Variables

var CategoryMap = make(map[string]int) // Maps Intel Category string to corresponding RowID in DB

var DefaultClient = NewClient()

var IDB_Http_Client = http.Client{Timeout: time.Duration(2) * time.Second}

var PrivateIPBlocks []*net.IPNet

var ThreatDBFile = "threats.db"

var UseIntel = false

Functions

func BufferFromFile

func BufferFromFile(inputfile *os.File) (*bufio.Reader, error)

func BuildThreatDB

func BuildThreatDB(arguments map[string]any, logger zerolog.Logger) error

func CheckIPinTI

func CheckIPinTI(ip string, isDataCenter bool, db *sql.DB) (string, string, string, bool, error)

func CloseChannelWhenDone

func CloseChannelWhenDone(c chan []string, wg *lbtypes.WaitGroupCount)

CloseChannelWhenDone is a helper function for handling channel closure with a wait-group.

func CombineOutputs

func CombineOutputs(arguments map[string]any, logger zerolog.Logger) error

CombineOutputs works to combine all CSV outputs on a per-directory level, trying to match header columns if possible

func CopyFile

func CopyFile(src, dst string) error

func CreateOutput

func CreateOutput(outputFile string) (*os.File, error)

func DoDNSEnrichment

func DoDNSEnrichment(ipaddress string) (string, string)

DoDNSEnrichment provides all current hostname records associated with a Domain along with the base hostname+tld, if any exist

func DoDomainWhoisenrichment

func DoDomainWhoisenrichment(domain string) []string

DoDomainWhoisenrichment returns a slice representing enrichments from a Domain-based WhoIS lookup.

func DoIDBEnrichment

func DoIDBEnrichment(ipaddress string) []string

DoIDBEnrichment returns a slice representing enrichments from Shodan's InternetDB project using the provided IP Address as the enrichment target.

func DoIPWhoisEnrichment

func DoIPWhoisEnrichment(ipaddress string) []string

DoIPWhoisEnrichment returns a slice representing enrichments from an IP-based WhoIS lookup.

func DoesFileExist

func DoesFileExist(filename string) bool

func DownloadAuthenticatedFile

func DownloadAuthenticatedFile(logger zerolog.Logger, url string, filepath string, key string, user string, password string) (err error)

func DownloadFile

func DownloadFile(logger zerolog.Logger, url string, filepath string, key string) (err error)

func ExtractTarGz

func ExtractTarGz(gzipStream io.Reader, logger zerolog.Logger, dir string) error

func FileToSlice

func FileToSlice(filename string, logger zerolog.Logger) []string

FileToSlice reads a file and returns a slice representing lines in the file

func FindOrGetDBs

func FindOrGetDBs(arguments map[string]any, logger zerolog.Logger, apikey string) error

func FindTargetIndexInSlice

func FindTargetIndexInSlice(headers []string, targetCol string) int

FindTargetIndexInSlice receives a string-slice and attempts to locate a specific value, returning the index of said value if it exists of -1 if not.

func GetFeedIDIfExist

func GetFeedIDIfExist(feed_name string, db *sql.DB) int

func GetHeaders

func GetHeaders(tempArgs map[string]any, headers []string) []string

func GetNewPW

func GetNewPW(logger zerolog.Logger, inputFile string, outputFile string) (*csv.Reader, *csv.Writer, *os.File, *os.File, error)

func Increment

func Increment(ip net.IP)

Increment increments the given net.IP by one bit. Incrementing the last IP in an IP space (IPv4, IPV6) is undefined.

func IngestFile

func IngestFile(inputFile string, categories string, feedid int, db *sql.DB, logger zerolog.Logger) error

func IngestIPNetLists

func IngestIPNetLists(url string, name string, file string, listtype string, category string, logger zerolog.Logger)

func InsertCategory

func InsertCategory(category string, db *sql.DB) error

func InsertFeed

func InsertFeed(feed_name string, feed_url string, db *sql.DB) (error, int)

func IntSlicetoStringSlice

func IntSlicetoStringSlice(s []int) []string

IntSlicetoStringSlice converts a slice of ints to a slice of the same length but of type string

func IsASN

func IsASN(s string) bool

func IsPrivateIP

func IsPrivateIP(ip net.IP, ipstring string) bool

func ListenOnWriteChannel

func ListenOnWriteChannel(c chan []string, w *csv.Writer, logger zerolog.Logger, outputF *os.File, bufferSize int, wait *lbtypes.WaitGroupCount)

func LookupIPRecords

func LookupIPRecords(ip string) []string

func OpenDBConnection

func OpenDBConnection(logger zerolog.Logger) (*sql.DB, error)

func OpenInput

func OpenInput(inputFile string) (*os.File, error)

func ProcessRecords

func ProcessRecords(logger zerolog.Logger, records [][]string, asnDB maxminddb.Reader, cityDB maxminddb.Reader, countryDB maxminddb.Reader, domainDB maxminddb.Reader, ipAddressColumn int, jsonColumn int, useRegex bool, useDNS bool, channel chan []string, waitGroup *lbtypes.WaitGroupCount, tracker *lbtypes.RunningJobs, tempArgs map[string]any, dateindex int)

func RegexFirstPublicIPFromString

func RegexFirstPublicIPFromString(input string) (string, bool)

RegexFirstPublicIPFromString attempts to use regex patterns to extract the first-identified non-private IP address from a string

func RemoveSpace

func RemoveSpace(s string) string

func ScannerFromFile

func ScannerFromFile(reader io.Reader) (*bufio.Scanner, error)

func SetAPIUrls

func SetAPIUrls(arguments map[string]any, logger zerolog.Logger) (error, string)

func SetupLogger

func SetupLogger() zerolog.Logger

func SetupPrivateNetworks

func SetupPrivateNetworks() error

func SummarizeThreatDB

func SummarizeThreatDB(logger zerolog.Logger)

func SummarizeThreatFeeds

func SummarizeThreatFeeds(logger zerolog.Logger)

func UpdateDCList

func UpdateDCList(logger zerolog.Logger)

func UpdateVPNList

func UpdateVPNList(logger zerolog.Logger)

func Whois

func Whois(domain string, servers ...string) (result string, err error)

Types

type Client

type Client struct {
	// contains filtered or unexported fields
}

func NewClient

func NewClient() *Client

func (*Client) Whois

func (c *Client) Whois(domain string, servers ...string) (result string, err error)

type Feed

type Feed struct {
	Name string   `json:"name"`
	URL  string   `json:"url"`
	Type []string `json:"type"`
}

type Feeds

type Feeds struct {
	Feeds []Feed `json:"feeds"`
}

type IPNetGenerator

type IPNetGenerator struct {
	*net.IPNet
	// contains filtered or unexported fields
}

IPNetGenerator is a net.IPnet wrapper that you can iterate over

func NewFromIPNet

func NewFromIPNet(ipNet *net.IPNet) *IPNetGenerator

NewFromIPNet creates a new IPNetGenerator from a *net.IPNet

func NewIPNetGenerator

func NewIPNetGenerator(cidr string) (*IPNetGenerator, error)

NewIPNetGenerator creates a new IPNetGenerator from a CIDR string, or an error if the CIDR is invalid.

func (*IPNetGenerator) Next

func (g *IPNetGenerator) Next() net.IP

Next returns the next net.IP in the subnet

type IPWhoisResult

type IPWhoisResult struct {
	NetRange            string
	CIDR                string
	NetName             string
	NetHandle           string
	Parent              string
	NetType             string
	OriginAS            string
	Customer            string
	RegistrationDate    string
	RegistrationUpdated string
	ReferenceURL        string
	CustomerName        string
	Address             string
	City                string
	StateProv           string
	PostalCode          string
	Country             string
	AddressUpdated      string
	EntityReferenceURL  string
	OrgNOCName          string
	OrgNOCEmail         string
	OrgTechName         string
	OrgTechEmail        string
	OrgAbuseName        string
	OrgAbuseEmail       string
}

func ParseIPWhoisLookup

func ParseIPWhoisLookup(data string) (IPWhoisResult, error)

ParseIPWhoisLookup parses the raw results of a Whois lookup against an IP Address