dumper

Documentation

Overview

Copyright 2023 The Chromium Authors Use of this source code is governed by a BSD-style license that can be found in the LICENSE file.

Copyright 2023 The Chromium Authors Use of this source code is governed by a BSD-style license that can be found in the LICENSE file.

Copyright 2024 The Chromium Authors Use of this source code is governed by a BSD-style license that can be found in the LICENSE file.

Index

• Constants
• Variables
• func IndexAssets(ctx context.Context) error
• func IndexMachines(ctx context.Context) error
• func InitServer(srv *server.Server) error
• func InstallCronServices(apiServer *server.Server)
• func SyncAssetInfoFromHaRT(ctx context.Context) error
• func TriggerJob(name string) error
• func Use(ctx context.Context, bqClient *bigquery.Client) context.Context
• func UseProject(ctx context.Context, project string) context.Context
• type CronServerImpl
  • func (c *CronServerImpl) TriggerCronJob(ctx context.Context, job *api.TriggerCronJobReq) (*empty.Empty, error)
• type DroneQueenClientGenerator

Constants

const (
	// Bucket From which GoldenEye data for devices is read
	GoldenEyeBucketName = "chromeos-build-release-console"
	GoldenEyeObjectName = "all_devices.json"
)

Variables

var DatastoreNamespaceToBigQueryDataset = map[string]string{
	util.BrowserNamespace:   ufsDatasetName,
	util.OSNamespace:        ufsDatasetName,
	util.OSPartnerNamespace: sfpDatasetName,
}

DatastoreNamespaceToBigQueryDataset refers a map between client namespace(set in context metadata) to BigQuery dataset

var Jobs = []*cron.CronTab{
	{

		Name:     "ufs.dumper.daily",
		Time:     20 * time.Minute,
		TrigType: cron.DAILY,
		Job:      dumpDaily,
	},
	{

		Name:     "ufs.dumper.hourly",
		Time:     30 * time.Minute,
		TrigType: cron.HOURLY,
		Job:      dumpHourly,
	},
	{

		Name:     "ufs.change_event.BqDump",
		Time:     10 * time.Minute,
		TrigType: cron.EVERY,
		Job:      dumpChangeEvent,
	},
	{

		Name:     "ufs.snapshot_msg.BqDump",
		Time:     10 * time.Minute,
		TrigType: cron.EVERY,
		Job:      dumpChangeSnapshots,
	},
	{

		Name:     "ufs.cros_network.dump",
		Time:     60 * time.Minute,
		TrigType: cron.EVERY,
		Job:      dumpCrosNetwork,
	},
	{

		Name:     "ufs.sync_devices.sync",
		TrigType: cron.HOURLY,
		Job:      SyncAssetInfoFromHaRT,
	},
	{

		Name:     "ufs.push_to_drone_queen",
		Time:     10 * time.Minute,
		TrigType: cron.EVERY,
		Job:      pushToDroneQueen,
	},
	{

		Name:     "ufs.report_inventory",
		Time:     5 * time.Minute,
		TrigType: cron.EVERY,
		Job:      reportUFSInventoryCronHandler,
	},
	{

		Name:     "ufs.report_bots",
		Time:     5 * time.Minute,
		TrigType: cron.EVERY,
		Job:      reportSwarmingBotsCronHandler,
	},
	{

		Name:     "ufs.sync_goldeneye_devices.sync",
		Time:     12 * time.Hour,
		TrigType: cron.EVERY,
		Job:      getGoldenEyeData,
	},
	{

		Name:     "ufs.swarming_labels_diff",
		Time:     5 * time.Minute,
		TrigType: cron.DAILY,
		Job:      swarmingLabelsDiffHandler,
	},
	{

		Name:     "ufs.sync_bot_config.sync",
		Time:     10 * time.Minute,
		TrigType: cron.EVERY,
		Job:      getBotConfigs,
	},
	{
		Name:     "ufs.device_config.sync",
		Time:     1 * time.Hour,
		TrigType: cron.EVERY,
		Job:      syncDeviceConfigs,
	},
	{

		Name:     "ufs.indexer.asset",
		Time:     200 * 365 * 24 * time.Hour,
		TrigType: cron.EVERY,
		Job:      IndexAssets,
	},
	{

		Name:     "ufs.indexer.machine",
		Time:     200 * 365 * 24 * time.Hour,
		TrigType: cron.EVERY,
		Job:      IndexMachines,
	},
	{

		Name:     "ufs.indexer.rack",
		Time:     200 * 365 * 24 * time.Hour,
		TrigType: cron.EVERY,
		Job:      indexRacks,
	},
	{

		Name:     "ufs.indexer.machinelse",
		Time:     200 * 365 * 24 * time.Hour,
		TrigType: cron.EVERY,
		Job:      indexMachineLSEs,
	},
	{

		Name:     "ufs.indexer.dutstate",
		Time:     200 * 365 * 24 * time.Hour,
		TrigType: cron.EVERY,
		Job:      indexDutStates,
	},
	{

		Name:     "ufs.backfill.devicelabels_machinelse",
		Time:     200 * 365 * 24 * time.Hour,
		TrigType: cron.EVERY,
		Job:      backfillDeviceLabelsMachineLSEs,
	},
	{

		Name:     "ufs.backfill.devicelabels_vm",
		Time:     200 * 365 * 24 * time.Hour,
		TrigType: cron.EVERY,
		Job:      backfillDeviceLabelsVMs,
	},
	{

		Name:     "ufs.backfill.devicelabels_schedulingunit",
		Time:     200 * 365 * 24 * time.Hour,
		TrigType: cron.EVERY,
		Job:      backfillDeviceLabelsSchedulingUnits,
	},
}

Jobs is a list of all the cron jobs that are currently available for running

Functions

func IndexAssets

func IndexAssets(ctx context.Context) error

IndexAssets updates the asset table thereby reindexing any new indexes that might be added to it. It is meant to be run during low-traffic/maintenance times as it attempts to index the entire table.

func IndexMachines

func IndexMachines(ctx context.Context) error

IndexMachines updates the machine table thereby reindexing any new indexes that might be added to it. It is meant to be run during low-traffic/maintenance times as it attempts to index the entire table.

func InitServer

func InitServer(srv *server.Server) error

InitServer initializes a cron server.

func InstallCronServices

func InstallCronServices(apiServer *server.Server)

InstallCronServices installs ...

func SyncAssetInfoFromHaRT

func SyncAssetInfoFromHaRT(ctx context.Context) error

SyncAssetInfoFromHaRT publishes the request for asset info to HaRT.

The response for this request will be made to an endpoint on an RPC call. This function only checks for the assets that have Device info missing or the last update on the device was 48 hours ago.

func TriggerJob

func TriggerJob(name string) error

TriggerJob triggers a job by name. Returns error if the job is not found.

func Use

func Use(ctx context.Context, bqClient *bigquery.Client) context.Context

Use installs bigquery client to context.

func UseProject

func UseProject(ctx context.Context, project string) context.Context

UseProject installs project name to context.

Types

type CronServerImpl

type CronServerImpl struct {
}

func (*CronServerImpl) TriggerCronJob

func (c *CronServerImpl) TriggerCronJob(ctx context.Context, job *api.TriggerCronJobReq) (*empty.Empty, error)

TriggerCronJob triggers the given cron job to run. Fails if the Job is already running.

type DroneQueenClientGenerator

type DroneQueenClientGenerator func(context.Context) (dronequeenapi.InventoryProviderClient, error)