jobs

Documentation

Index

• Constants
• Variables
• func CheckLogSTH(qc *que.Client, logger *log.Logger, job *que.Job, tx *pgx.Tx) error
• func GetEntries(qc *que.Client, logger *log.Logger, job *que.Job, tx *pgx.Tx) error
• func NewLogMetadata(qc *que.Client, logger *log.Logger, job *que.Job, tx *pgx.Tx) error
• func RefreshMetadataForEntries(qc *que.Client, logger *log.Logger, job *que.Job, tx *pgx.Tx) error
• func UpdateCTLogList(qc *que.Client, logger *log.Logger, job *que.Job, tx *pgx.Tx) error
• type CTLog
• type CheckSTHConf
• type GetEntriesConf
• type UpdateDataGovAU
  • func (us *UpdateDataGovAU) BackfillDataGovAU(qc *que.Client, logger *log.Logger, job *que.Job, tx *pgx.Tx) error
  • func (us *UpdateDataGovAU) InsertRecords(logger *log.Logger, recs []*ckanRecord) error
  • func (us *UpdateDataGovAU) Run(qc *que.Client, logger *log.Logger, job *que.Job, tx *pgx.Tx) error
• type UpdateDataGovAUConf
• type UpdateSlack
  • func (us *UpdateSlack) Run(qc *que.Client, logger *log.Logger, job *que.Job, tx *pgx.Tx) error
• type UpdateSlackConf

Constants

const (
	// KeyCheckSTH is the name of the job
	KeyCheckSTH = "cron_check_sth"

	// InsecurePrefix can be prepended to the "connect_url" field to indicate that TLS verification should be disabled for a particular URL.
	// This is used from some older logs that appear to still be up, but have issues with their certificates.
	InsecurePrefix = "insecure-skip-verify-"
)

const (
	KeyGetEntries     = "get_entries"
	KeyUpdateMetadata = "update_metadata"

	DomainSuffix = ".gov.au"
	MatchDomain  = "gov.au"

	MaxToRequest = 1024

	MaxToUpdate = 1024
)

const (
	KeyUpdateDataGovAU   = "update_data_gov_au"
	KeyBackfillDataGovAU = "backfill_data_gov_au"
	MaxToBackfill        = 256
)

const (
	StateActive = 0
	StateIgnore = 1
)

const (
	KeyUpdateLogs = "cron_update_logs"

	KnownLogsURL = "https://www.gstatic.com/ct/log_list/all_logs_list.json"
)

const (
	KeyNewLogMetadata = "new_log_metadata"
)

const (
	KeyUpdateSlack = "cron_slack"
)

Variables

var (
	Jurisdictions = map[string]string{
		"TAS": "tas.gov.au",
		"VIC": "vic.gov.au",
		"NSW": "nsw.gov.au",
		"QLD": "qld.gov.au",
		"WA":  "wa.gov.au",
		"SA":  "sa.gov.au",
		"NT":  "nt.gov.au",
		"ACT": "act.gov.au",
	}

	CDNs = map[string]string{
		"cloudflaressl": "CloudFlare",
		"incapsula":     "Incapsula",
		"fastly":        "Fastly",
		"pantheonsite":  "PantheonSite",
	}
)

Functions

func CheckLogSTH

func CheckLogSTH(qc *que.Client, logger *log.Logger, job *que.Job, tx *pgx.Tx) error

CheckLogSTH checks for new entries, and schedules a job to fetch them if needed

func GetEntries

func GetEntries(qc *que.Client, logger *log.Logger, job *que.Job, tx *pgx.Tx) error

func NewLogMetadata

func NewLogMetadata(qc *que.Client, logger *log.Logger, job *que.Job, tx *pgx.Tx) error

func RefreshMetadataForEntries

func RefreshMetadataForEntries(qc *que.Client, logger *log.Logger, job *que.Job, tx *pgx.Tx) error

func UpdateCTLogList

func UpdateCTLogList(qc *que.Client, logger *log.Logger, job *que.Job, tx *pgx.Tx) error

Types

type CTLog

type CTLog struct {
	URL            string `json:"url"`
	DisqualifiedAt int64  `json:"disqualified_at"`
	FinalSTH       *struct {
		TreeSize int64 `json:"tree_size"`
	} `json:"final_sth"`
}

type CheckSTHConf

type CheckSTHConf struct {
	// URL is used to lookup the record in the monitored_logs table
	URL string
}

CheckSTHConf is stored in the que_jobs table

type GetEntriesConf

type GetEntriesConf struct {
	URL        string
	Start, End uint64 // end is exclusive
}

type UpdateDataGovAU

type UpdateDataGovAU struct {
	BaseURL    string
	APIKey     string
	ResourceID string
}

func (*UpdateDataGovAU) BackfillDataGovAU

func (us *UpdateDataGovAU) BackfillDataGovAU(qc *que.Client, logger *log.Logger, job *que.Job, tx *pgx.Tx) error

func (*UpdateDataGovAU) InsertRecords

func (us *UpdateDataGovAU) InsertRecords(logger *log.Logger, recs []*ckanRecord) error

func (*UpdateDataGovAU) Run

func (us *UpdateDataGovAU) Run(qc *que.Client, logger *log.Logger, job *que.Job, tx *pgx.Tx) error

type UpdateDataGovAUConf

type UpdateDataGovAUConf struct {
	Data []byte
}

type UpdateSlack

type UpdateSlack struct {
	BaseURL string
	Hook    string
}

func (*UpdateSlack) Run

func (us *UpdateSlack) Run(qc *que.Client, logger *log.Logger, job *que.Job, tx *pgx.Tx) error

type UpdateSlackConf

type UpdateSlackConf struct {
	Key     string
	Domains []string
	Issuer  string
}