util

package
v0.0.0-...-1ea9000 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Dec 20, 2024 License: BSD-3-Clause Imports: 43 Imported by: 37

Documentation

Overview

Utility to create and manage chromium builds.

Utility that contains functions to interact with ct-perf.skia.org.

Google Storage utility that contains methods for both CT master and worker scripts.

Utilities for running on a local machine.

Utility that contains methods for both CT master and worker scripts.

Index

Constants

View Source 
const (
	// Use the CTFE proxy to Google Storage. See skbug.com/6762
	GCS_HTTP_LINK = "https://ct.skia.org/results/"

	// File names and dir names.
	CHROMIUM_BUILDS_DIR_NAME         = "chromium_builds"
	PAGESETS_DIR_NAME                = "page_sets"
	WEB_ARCHIVES_DIR_NAME            = "webpage_archives"
	STORAGE_DIR_NAME                 = "storage"
	REPO_DIR_NAME                    = "skia-repo"
	TASKS_DIR_NAME                   = "tasks"
	BINARIES_DIR_NAME                = "binaries"
	BENCHMARK_TASKS_DIR_NAME         = "benchmark_runs"
	CHROMIUM_PERF_TASKS_DIR_NAME     = "chromium_perf_runs"
	CHROMIUM_ANALYSIS_TASKS_DIR_NAME = "chromium_analysis_runs"
	FIX_ARCHIVE_TASKS_DIR_NAME       = "fix_archive_runs"
	TRACE_DOWNLOADS_DIR_NAME         = "trace_downloads"
	CHROMIUM_BUILD_ZIP_NAME          = "chromium_build.zip"
	CUSTOM_APK_DIR_NAME              = "custom-apk"

	// Limit the number of times CT tries to get a remote file before giving up.
	MAX_URI_GET_TRIES = 4

	// Pageset types supported by CT.
	PAGESET_TYPE_ALL                 = "All"
	PAGESET_TYPE_100k                = "100k"
	PAGESET_TYPE_MOBILE_100k         = "Mobile100k"
	PAGESET_TYPE_LAYOUTSHIFT_100k    = "LayoutShift100k"
	PAGESET_TYPE_10k                 = "10k"
	PAGESET_TYPE_MOBILE_10k          = "Mobile10k"
	PAGESET_TYPE_MOBILE_VOLT_10k     = "VoltMobile10k"
	PAGESET_TYPE_LAYOUTSHIFT_10k     = "LayoutShift10k"
	PAGESET_TYPE_AMP_LIVE_REPRO      = "AMPLiveRepro"
	PAGESET_TYPE_AMP_PUPPETEER_SITES = "AMPPuppeteerSites"
	PAGESET_TYPE_DUMMY_1k            = "Dummy1k"       // Used for testing.
	PAGESET_TYPE_MOBILE_DUMMY_1k     = "DummyMobile1k" // Used for testing.

	// Names of binaries executed by CT.
	BINARY_CHROME          = "chrome"
	BINARY_CHROME_WINDOWS  = "chrome.exe"
	BINARY_RECORD_WPR      = "record_wpr"
	BINARY_RUN_BENCHMARK   = "run_benchmark"
	BINARY_ANALYZE_METRICS = "analyze_metrics_ct.py"
	BINARY_GCLIENT         = "gclient"
	BINARY_NINJA           = "ninja"
	BINARY_ADB             = "adb"
	BINARY_MAIL            = "mail"
	BINARY_PYTHON          = "python3"
	BINARY_VPYTHON3        = "vpython3"

	// Platforms supported by CT.
	PLATFORM_ANDROID = "Android"
	PLATFORM_LINUX   = "Linux"
	PLATFORM_WINDOWS = "Windows"

	// Benchmarks supported by CT.
	BENCHMARK_RR                       = "rasterize_and_record_micro_ct"
	BENCHMARK_LOADING                  = "loading.cluster_telemetry"
	BENCHMARK_SCREENSHOT               = "screenshot_ct"
	BENCHMARK_RENDERING                = "rendering.cluster_telemetry"
	BENCHMARK_USECOUNTER               = "usecounter_ct"
	BENCHMARK_LEAK_DETECTION           = "leak_detection.cluster_telemetry"
	BENCHMARK_MEMORY                   = "memory.cluster_telemetry"
	BENCHMARK_V8_LOADING               = "v8.loading.cluster_telemetry"
	BENCHMARK_V8_LOADING_RUNTIME_STATS = "v8.loading_runtime_stats.cluster_telemetry"
	BENCHMARK_GENERIC_TRACE            = "generic_trace_ct"
	BENCHMARK_AD_TAGGING               = "ad_tagging.cluster_telemetry"
	BENCHMARK_LAYOUT_SHIFT             = "layout_shift.cluster_telemetry"

	// Default browser args when running benchmarks.
	DEFAULT_BROWSER_ARGS = ""
	// Default value column name to use when merging CSVs.
	DEFAULT_VALUE_COLUMN_NAME = "avg"

	// Use live sites flag.
	USE_LIVE_SITES_FLAGS = "--use-live-sites"
	// Pageset repeat flag.
	PAGESET_REPEAT_FLAG = "--pageset-repeat"
	// User agent flag.
	USER_AGENT_FLAG = "--user-agent"
	// Run Benchmark timeout flag.
	RUN_BENCHMARK_TIMEOUT_FLAG = "--run-benchmark-timeout"
	// Max pages per bot flag.
	MAX_PAGES_PER_BOT = "--max-pages-per-bot"
	// Num of retries used by analysis task.
	NUM_ANALYSIS_RETRIES = "--num-analysis-retries"

	// Defaults for custom webpages.
	DEFAULT_CUSTOM_PAGE_ARCHIVEPATH = "dummy_path"

	PKILL_TIMEOUT              = 5 * time.Minute
	HTTP_CLIENT_TIMEOUT        = 30 * time.Minute
	FETCH_GN_TIMEOUT           = 2 * time.Minute
	GN_GEN_TIMEOUT             = 2 * time.Minute
	UPDATE_DEPOT_TOOLS_TIMEOUT = 5 * time.Minute

	// util.SyncDir
	GIT_PULL_TIMEOUT     = 30 * time.Minute
	GCLIENT_SYNC_TIMEOUT = 30 * time.Minute

	// util.ResetCheckout
	GIT_CHECKOUT_TIMEOUT = 10 * time.Minute
	GIT_REBASE_TIMEOUT   = 10 * time.Minute
	GIT_RESET_TIMEOUT    = 10 * time.Minute
	GIT_CLEAN_TIMEOUT    = 10 * time.Minute

	// util.CreateChromiumBuildOnSwarming
	SYNC_SKIA_IN_CHROME_TIMEOUT = 2 * time.Hour
	GIT_LS_REMOTE_TIMEOUT       = 5 * time.Minute
	GIT_APPLY_TIMEOUT           = 5 * time.Minute
	GN_CHROMIUM_TIMEOUT         = 30 * time.Minute
	NINJA_TIMEOUT               = 3 * time.Hour

	// util.UnInstallChromeAPK
	ADB_UNINSTALL_TIMEOUT = time.Minute

	// util.InstallChromeAPK
	ADB_INSTALL_TIMEOUT = 5 * time.Minute

	// Capture Archives
	CAPTURE_ARCHIVES_DEFAULT_CT_BENCHMARK = "rasterize_and_record_micro_ct"
	CAPTURE_ARCHIVES_AMP_STORY            = "layout_shift_cluster_telemetry"

	// Run Chromium Perf
	ADB_VERSION_TIMEOUT            = 5 * time.Minute
	ADB_ROOT_TIMEOUT               = 5 * time.Minute
	CSV_PIVOT_TABLE_MERGER_TIMEOUT = 30 * time.Minute
	CSV_MERGER_TIMEOUT             = 1 * time.Hour
	CSV_COMPARER_TIMEOUT           = 2 * time.Hour

	// Poller
	MAKE_ALL_TIMEOUT = 15 * time.Minute

	// Swarming constants.
	SWARMING_DIR_NAME               = "swarming"
	SWARMING_POOL                   = "CT"
	BUILD_OUTPUT_FILENAME           = "build_remote_dirs.txt"
	ISOLATE_TELEMETRY_FILENAME      = "isolate_telemetry_hash.txt"
	MAX_SWARMING_HARD_TIMEOUT_HOURS = 24
	// Timeouts.
	BATCHARCHIVE_TIMEOUT = 10 * time.Minute
	XVFB_TIMEOUT         = 5 * time.Minute

	// Swarming links and params.
	// TODO(rmistry): The below link contains "st=1262304000000" which is from 2010. This is done so
	// that swarming will not use today's timestamp as default. See if there is a better way to handle
	// this.
	SWARMING_RUN_ID_ALL_TASKS_LINK_TEMPLATE   = "https://chrome-swarming.appspot.com/tasklist?l=500&c=name&c=created_ts&c=bot&c=duration&c=state&f=runid:%s&st=1262304000000"
	SWARMING_RUN_ID_TASK_LINK_PREFIX_TEMPLATE = SWARMING_RUN_ID_ALL_TASKS_LINK_TEMPLATE + "&f=name:%s"

	// Priorities
	TASKS_PRIORITY_HIGH   = swarming.RECOMMENDED_PRIORITY
	TASKS_PRIORITY_MEDIUM = swarming.RECOMMENDED_PRIORITY + 10
	TASKS_PRIORITY_LOW    = swarming.RECOMMENDED_PRIORITY + 20

	// ct-perf.skia.org constants.
	CT_PERF_BUCKET = "cluster-telemetry-perf"
	CT_PERF_REPO   = "https://skia.googlesource.com/perf-ct"

	// The CT service account is unfortunately named, it runs on not just Golo
	// bots but also on GCE instances. Would be nice to rename this one day following
	// the steps in https://bugs.chromium.org/p/skia/issues/detail?id=10187#c1
	CT_SERVICE_ACCOUNT = "ct-golo@ct-swarming-bots.iam.gserviceaccount.com"

	CHROME_ANDROID_PACKAGE_NAME  = "com.google.android.apps.chrome"
	ADB_CIPD_PACKAGE             = "cipd_bin_packages:infra/adb/linux-amd64:adb_version:1.0.36"
	LUCI_AUTH_CIPD_PACKAGE_LINUX = "cipd_bin_packages:infra/tools/luci-auth/linux-amd64:git_revision:41a7e9bcbf18718dcda83dd5c6188cfc44271e70"
	LUCI_AUTH_CIPD_PACKAGE_WIN   = "cipd_bin_packages:infra/tools/luci-auth/windows-amd64:git_revision:41a7e9bcbf18718dcda83dd5c6188cfc44271e70"
)
View Source 
const (
	DOWNLOAD_UPLOAD_GOROUTINE_POOL_SIZE = 30
	// Use larger pool size for deletions. This is useful when deleting directories
	// with 1M/1B subdirectories from the master. Google Storage will not be overwhelmed
	// because all workers do not do large scale deletions at the same time.
	DELETE_GOROUTINE_POOL_SIZE = 1000
)
View Source 
const (
	MAX_SYNC_TRIES = 3

	TS_FORMAT = "20060102150405"

	MAX_SIMULTANEOUS_SWARMING_TASKS_PER_RUN = 10000

	PATCH_LIMIT = 1 << 26
)
View Source 
const (
	// Use 14 chars here instead of the traditional 7 to reduce the chances of
	// ambiguous hashes while still leaving directory lengths reasonable.
	TRUNCATED_HASH_LENGTH = 14
)

Variables

View Source 
var (
	TELEMETRY_ISOLATES_TARGET  = "ct_telemetry_perf_tests_without_chrome"
	TELEMETRY_ISOLATES_OUT_DIR = filepath.Join("out", "telemetry_isolates")
)
View Source 
var (
	CtUser = "chrome-bot"
	// Whenever the bucket name changes, getGSLink in ctfe.js will have to be
	// updated as well.
	GCSBucketName = "cluster-telemetry"

	// Email address of cluster telemetry admins. They will be notified every time
	// a task has started and completed.
	CtAdmins = []string{"rmistry@google.com"}

	// Names of local directories and files.
	StorageDir             = filepath.Join("/", "b", STORAGE_DIR_NAME)
	RepoDir                = filepath.Join("/", "b", REPO_DIR_NAME)
	DepotToolsDir          = filepath.Join("/", "home", "chrome-bot", "depot_tools")
	ChromiumBuildsDir      = filepath.Join(StorageDir, CHROMIUM_BUILDS_DIR_NAME)
	ChromiumSrcDir         = filepath.Join(StorageDir, "chromium", "src")
	TelemetryBinariesDir   = filepath.Join(ChromiumSrcDir, "tools", "perf")
	TelemetrySrcDir        = filepath.Join(ChromiumSrcDir, "tools", "telemetry")
	RelativeCatapultSrcDir = filepath.Join("third_party", "catapult")
	CatapultSrcDir         = filepath.Join(ChromiumSrcDir, RelativeCatapultSrcDir)
	V8SrcDir               = filepath.Join(ChromiumSrcDir, "v8")
	TaskFileDir            = filepath.Join(StorageDir, "current_task")
	GCSTokenPath           = filepath.Join(StorageDir, "google_storage_token.data")
	PagesetsDir            = filepath.Join(StorageDir, PAGESETS_DIR_NAME)
	WebArchivesDir         = filepath.Join(StorageDir, WEB_ARCHIVES_DIR_NAME)
	ApkName                = "ChromePublic.apk"
	SkiaTreeDir            = filepath.Join(RepoDir, "trunk")
	CtTreeDir              = filepath.Join(RepoDir, "go", "src", "go.skia.org", "infra", "ct")

	// Names of local and remote directories and files.
	BinariesDir                    = filepath.Join(BINARIES_DIR_NAME)
	BenchmarkRunsDir               = filepath.Join(TASKS_DIR_NAME, BENCHMARK_TASKS_DIR_NAME)
	BenchmarkRunsStorageDir        = path.Join(TASKS_DIR_NAME, BENCHMARK_TASKS_DIR_NAME)
	ChromiumPerfRunsDir            = filepath.Join(TASKS_DIR_NAME, CHROMIUM_PERF_TASKS_DIR_NAME)
	ChromiumPerfRunsStorageDir     = path.Join(TASKS_DIR_NAME, CHROMIUM_PERF_TASKS_DIR_NAME)
	ChromiumAnalysisRunsStorageDir = path.Join(TASKS_DIR_NAME, CHROMIUM_ANALYSIS_TASKS_DIR_NAME)
	FixArchivesRunsDir             = filepath.Join(TASKS_DIR_NAME, FIX_ARCHIVE_TASKS_DIR_NAME)
	TraceDownloadsDir              = filepath.Join(TASKS_DIR_NAME, TRACE_DOWNLOADS_DIR_NAME)

	// Information about the different CT pageset types.
	PagesetTypeToInfo = map[string]*PagesetTypeInfo{
		PAGESET_TYPE_ALL: {
			NumPages:                   1000000,
			CSVSource:                  "csv/top-1m.csv",
			UserAgent:                  "desktop",
			CreatePagesetsTimeoutSecs:  1800,
			CaptureArchivesTimeoutSecs: 300,
			RunChromiumPerfTimeoutSecs: 300,
			Description:                "Top 1M (with desktop user-agent)",
		},
		PAGESET_TYPE_100k: {
			NumPages:                   100000,
			CSVSource:                  "csv/top-1m.csv",
			UserAgent:                  "desktop",
			CreatePagesetsTimeoutSecs:  1800,
			CaptureArchivesTimeoutSecs: 300,
			RunChromiumPerfTimeoutSecs: 300,
			Description:                "Top 100K (with desktop user-agent)",
		},
		PAGESET_TYPE_MOBILE_100k: {
			NumPages:                   100000,
			CSVSource:                  "csv/android-top-1m.csv",
			UserAgent:                  "mobile",
			CreatePagesetsTimeoutSecs:  1800,
			CaptureArchivesTimeoutSecs: 300,
			RunChromiumPerfTimeoutSecs: 300,
			Description:                "Top 100K (with mobile user-agent)",
		},
		PAGESET_TYPE_LAYOUTSHIFT_100k: {
			NumPages:                   100000,
			CSVSource:                  "csv/layout-shift-100k.csv",
			UserAgent:                  "desktop",
			CreatePagesetsTimeoutSecs:  1800,
			CaptureArchivesTimeoutSecs: 300,
			RunChromiumPerfTimeoutSecs: 300,
			Description:                "Layout Shift 100K (with desktop user-agent)",
		},
		PAGESET_TYPE_10k: {
			NumPages:                   10000,
			CSVSource:                  "csv/top-1m.csv",
			UserAgent:                  "desktop",
			CreatePagesetsTimeoutSecs:  1800,
			CaptureArchivesTimeoutSecs: 300,
			RunChromiumPerfTimeoutSecs: 300,
			Description:                "Top 10K (with desktop user-agent)",
		},
		PAGESET_TYPE_MOBILE_10k: {
			NumPages:                   10000,
			CSVSource:                  "csv/android-top-1m.csv",
			UserAgent:                  "mobile",
			CreatePagesetsTimeoutSecs:  1800,
			CaptureArchivesTimeoutSecs: 300,
			RunChromiumPerfTimeoutSecs: 300,
			Description:                "Top 10K (with mobile user-agent)",
		},
		PAGESET_TYPE_MOBILE_VOLT_10k: {
			NumPages:                   10000,
			CSVSource:                  "csv/volt-10k.csv",
			UserAgent:                  "mobile",
			CreatePagesetsTimeoutSecs:  1800,
			CaptureArchivesTimeoutSecs: 300,
			RunChromiumPerfTimeoutSecs: 300,
			Description:                "Volt 10K (with mobile user-agent)",
		},
		PAGESET_TYPE_LAYOUTSHIFT_10k: {
			NumPages:                   10000,
			CSVSource:                  "csv/layout-shift-10k.csv",
			UserAgent:                  "desktop",
			CreatePagesetsTimeoutSecs:  1800,
			CaptureArchivesTimeoutSecs: 300,
			RunChromiumPerfTimeoutSecs: 300,
			Description:                "Layout Shift 10K (with desktop user-agent)",
		},
		PAGESET_TYPE_AMP_LIVE_REPRO: {
			NumPages:                   3600,
			CSVSource:                  "csv/amp-cls-ct-live-repro.csv",
			UserAgent:                  "mobile",
			CreatePagesetsTimeoutSecs:  1800,
			CaptureArchivesTimeoutSecs: 300,
			RunChromiumPerfTimeoutSecs: 300,
			Description:                "AMP live repro (with mobile user-agent)",
		},
		PAGESET_TYPE_AMP_PUPPETEER_SITES: {
			NumPages:                   1000,
			CSVSource:                  "csv/amp-cls-puppeteer-sites.csv",
			UserAgent:                  "mobile",
			CreatePagesetsTimeoutSecs:  1800,
			CaptureArchivesTimeoutSecs: 300,
			RunChromiumPerfTimeoutSecs: 300,
			Description:                "AMP puppeteer sites (with mobile user-agent)",
		},
		PAGESET_TYPE_DUMMY_1k: {
			NumPages:                   1000,
			CSVSource:                  "csv/top-1m.csv",
			UserAgent:                  "desktop",
			CreatePagesetsTimeoutSecs:  1800,
			CaptureArchivesTimeoutSecs: 300,
			RunChromiumPerfTimeoutSecs: 300,
			Description:                "Top 1K (with desktop user-agent, for testing, hidden from Runs History by default)",
		},
		PAGESET_TYPE_MOBILE_DUMMY_1k: {
			NumPages:                   1000,
			CSVSource:                  "csv/android-top-1m.csv",
			UserAgent:                  "mobile",
			CreatePagesetsTimeoutSecs:  1800,
			CaptureArchivesTimeoutSecs: 300,
			RunChromiumPerfTimeoutSecs: 300,
			Description:                "Top 1K (with mobile user-agent, for testing, hidden from Runs History by default)",
		},
	}

	// Frontend constants below.
	SupportedBenchmarksToDoc = map[string]string{
		BENCHMARK_RR:                       "https://cs.chromium.org/chromium/src/tools/perf/contrib/cluster_telemetry/rasterize_and_record_micro_ct.py",
		BENCHMARK_LOADING:                  "https://cs.chromium.org/chromium/src/tools/perf/contrib/cluster_telemetry/v8_loading_ct.py",
		BENCHMARK_USECOUNTER:               "https://docs.google.com/document/d/1FSzJm2L2ow6pZTM_CuyHNJecXuX7Mx3XmBzL4SFHyLA/",
		BENCHMARK_LEAK_DETECTION:           "https://docs.google.com/document/d/1wUWa7dWUdvr6dLdYHFfMQdnvgzt7lrrvzYfpAK-_6e0/",
		BENCHMARK_RENDERING:                "https://cs.chromium.org/chromium/src/tools/perf/contrib/cluster_telemetry/rendering_ct.py",
		BENCHMARK_MEMORY:                   "https://cs.chromium.org/chromium/src/tools/perf/contrib/cluster_telemetry/memory_ct.py",
		BENCHMARK_V8_LOADING:               "https://cs.chromium.org/chromium/src/tools/perf/contrib/cluster_telemetry/v8_loading_ct.py",
		BENCHMARK_V8_LOADING_RUNTIME_STATS: "https://cs.chromium.org/chromium/src/tools/perf/contrib/cluster_telemetry/v8_loading_runtime_stats_ct.py",
		BENCHMARK_GENERIC_TRACE:            "https://docs.google.com/document/d/1vGd7dnrxayMYHPO72wWkwTvjMnIRrel4yxzCr1bMiis/",
		BENCHMARK_AD_TAGGING:               "https://docs.google.com/document/d/1zlWQoLjGuYOWDR_vkVRYoVbU89JetNDOlcDuOaNAzDc/",
		BENCHMARK_LAYOUT_SHIFT:             "https://docs.google.com/document/d/1bYffpPHWFVaaAve2OZCFuv5xentVlFZF4GxZTaDLoXc/",
	}

	SupportedPlatformsToDesc = map[string]string{
		PLATFORM_LINUX:   "Linux (Ubuntu18.04 machines)",
		PLATFORM_ANDROID: "Android (Pixel2 devices)",
	}

	TaskPrioritiesToDesc = map[int]string{
		TASKS_PRIORITY_HIGH:   "High priority",
		TASKS_PRIORITY_MEDIUM: "Medium priority",
		TASKS_PRIORITY_LOW:    "Low priority",
	}

	// Swarming machine dimensions.
	GCE_LINUX_MASTER_DIMENSIONS = map[string]string{"pool": "CTMaster", "os": "Linux", "cores": "4"}

	GCE_LINUX_WORKER_DIMENSIONS   = map[string]string{"pool": SWARMING_POOL, "os": "Linux", "cores": "4"}
	GCE_WINDOWS_WORKER_DIMENSIONS = map[string]string{"pool": SWARMING_POOL, "os": "Windows", "cores": "4"}

	GCE_ANDROID_BUILDER_DIMENSIONS = map[string]string{"pool": "CTAndroidBuilder", "cores": "64"}
	GCE_LINUX_BUILDER_DIMENSIONS   = map[string]string{"pool": "CTLinuxBuilder", "cores": "64"}
	GCE_WINDOWS_BUILDER_DIMENSIONS = map[string]string{"pool": "CTBuilder", "os": "Windows"}

	GOLO_ANDROID_WORKER_DIMENSIONS = map[string]string{"pool": SWARMING_POOL, "os": "Android", "inside_docker": "1", "device_type": "walleye"}
	GOLO_LINUX_WORKER_DIMENSIONS   = map[string]string{"pool": SWARMING_POOL, "os": "Linux", "cores": "8"}

	// ct-perf.skia.org constants.
	CTPerfWorkDir = filepath.Join(StorageDir, "ct-perf-workdir")
)
View Source 
var (
	CIPD_PATHS = []string{
		"cipd_bin_packages",
		"cipd_bin_packages/bin",
		"cipd_bin_packages/cpython",
		"cipd_bin_packages/cpython/bin",
		"cipd_bin_packages/cpython3",
		"cipd_bin_packages/cpython3/bin",
	}

	VPYTHON_VIRTUALENV_ROOT = filepath.Join(os.TempDir(), "vpython")
)

Functions

func AddCTRunDataToPerf 

func AddCTRunDataToPerf(ctx context.Context, groupName, runID, pathToCSVResults, gitExec string, gs *GcsUtil) error

AddCTRunDataToPerf converts and uploads data from the CT run to CT's perf instance.

It does the following:

  1. Adds a commit to CT Perf's synthetic repo in https://skia.googlesource.com/perf-ct/+show/master
  2. Constructs a results struct in the format of https://skia.googlesource.com/buildbot/+doc/master/perf/FORMAT.md Ensures that the results struct has as key the runID, groupName and the git hash from (1). Populates the results struct using the output CSV file from CT's run.
  3. Create JSON file from the results struct.
  4. Uploads the results file to Google storage bucket CT_PERF_BUCKET for ingestion by ct-perf.skia.org. It is stored in location of this format: gs://<bucket>/<one or more dir names>/YYYY/MM/DD/HH/<zero or more dir names><some unique name>.json

For example, it converts the following example CSV file:

paint_op_count,traceUrls,pixels_rasterized,rasterize_time (ms),record_time_caching_disabled (ms),record_time_subsequence_caching_disabled (ms),painter_memory_usage (B),record_time_construction_disabled (ms),page_name 805.0,,1310720.0,2.449,1.128,0.283,25856.0,0.335,http://www.reuters.com (#480) 643.0,,1310720.0,2.894,0.998,0.209,24856.0,0.242,http://www.rediff.com (#490)

into 

{
  "gitHash" : "8dcc84f7dc8523dd90501a4feb1f632808337c34",
  "runID" : "rmistry-xyz",
  "key" : {
    "group_name" : "BGPT perf"
  },
  "results" : {
    "http://www.reuters.com" : {
      "default" : {
        "paint_op_count": 805.0,
        "pixels_rasterized": 1310720.0,
        "rasterize_time (ms)": 2.449,
        "record_time_caching_disabled (ms)": 1.128,
        "record_time_subsequence_caching_disabled (ms)": 0.283,
        "painter_memory_usage (B)": 25856.0,
        "record_time_construction_disabled (ms)": 0.335,
        "options" : {
          "page_rank" : 480,
        },
      },
    "http://www.rediff.com" : {
      "default" : {
        "paint_op_count": 643.0,
        "pixels_rasterized": 1310720.0,
        "rasterize_time (ms)": 2.894,
        "record_time_caching_disabled (ms)": 0.998,
        "record_time_subsequence_caching_disabled (ms)": 0.209,
        "painter_memory_usage (B)": 24856.0,
        "record_time_construction_disabled (ms)": 0.242,
        "options" : {
          "page_rank" : 490,
        },
      },
    }
  }
}

func ApplyPatch 

func ApplyPatch(ctx context.Context, patch, dir, gitExec string) error

ApplyPatch applies a patch to a Git checkout.

func ChromeProcessesCleaner 

func ChromeProcessesCleaner(ctx context.Context, locker sync.Locker, chromeCleanerTimer time.Duration)

Running benchmarks in parallel leads to multiple chrome instances coming up at the same time, when there are crashes chrome processes stick around which can severely impact the machine's performance. To stop this from happening chrome zombie processes are periodically killed.

func ChromiumBuildDir 

func ChromiumBuildDir(chromiumHash, skiaHash, runID string) string

Construct the name of a directory to store a chromium build. For generic clean builds, runID should be empty.

func CleanTmpDir 

func CleanTmpDir()

CleanTmpDir deletes all tmp files from the caller because telemetry tends to generate a lot of temporary artifacts there and they take up root disk space.

func CreateChromiumBuildOnSwarming 

func CreateChromiumBuildOnSwarming(ctx context.Context, runID, targetPlatform, chromiumHash, skiaHash, pathToPyFiles, gitExec, gnArgs string, applyPatches, uploadSingleBuild bool) (string, string, error)

runID is the unique id of the current run (typically requester + timestamp). targetPlatform is the platform the benchmark will run on (Android / Linux / Windows ). chromiumHash is the hash the checkout should be synced to. If not specified then Chromium's Tot hash is used. skiaHash is the hash the checkout should be synced to. If not specified then Skia's LKGR hash is used (the hash in Chromium's DEPS file). pathToPyFiles is the local path to CT's python scripts. Eg: sync_skia_in_chrome.py. gitExec is the local path to the git binary. applyPatches if true looks for Chromium/Skia/V8/Catapult patches in the temp dir and runs once with the patch applied and once without the patch applied. uploadSingleBuild if true does not upload a 2nd build of Chromium.

func CreateCustomPagesets 

func CreateCustomPagesets(webpages []string, pagesetsDir, targetPlatform string, startRange int) error

func CreateTelemetryIsolates 

func CreateTelemetryIsolates(ctx context.Context, runID, targetPlatform, chromiumHash, pathToPyFiles, gitExec string, applyPatches bool) error

CreateTelemetryIsolates creates an isolate of telemetry binaries that can be distributed to CT workers to run run_benchmark or record_wpr.

ctx is the Context to use. runID is the unique id of the current run (typically requester + timestamp). targetPlatform is the platform the benchmark will run on (Android / Linux / Windows). chromiumHash is the hash the checkout should be synced to. pathToPyFiles is the local path to CT's python scripts. Eg: sync_skia_in_chrome.py. gitExec is the local path to the git binary. applyPatches if true looks for Chromium/Skia/V8/Catapult patches in the temp dir.

func DownloadAndApplyPatch 

func DownloadAndApplyPatch(ctx context.Context, patchName, localDir, remotePatchesDir, checkout, gitExec string, gs *GcsUtil) error

func DownloadPatch 

func DownloadPatch(localPath, remotePath string, gs *GcsUtil) (int64, error)

func ExecuteCmd 

func ExecuteCmd(ctx context.Context, binary string, args, env []string, timeout time.Duration, stdout, stderr io.Writer) error

ExecuteCmd calls ExecuteCmdWithConfigurableLogging with logStdout and logStderr set to true.

func ExecuteCmdWithConfigurableLogging 

func ExecuteCmdWithConfigurableLogging(ctx context.Context, binary string, args, env []string, timeout time.Duration, stdout, stderr io.Writer, logStdout, logStderr bool) error

ExecuteCmdWithConfigurableLogging executes the specified binary with the specified args and env. Stdout and Stderr are written to stdout and stderr respectively if specified. If not specified then Stdout and Stderr will be outputted only to sklog. 

func GetAnalysisOutputLink(runID string) string

func GetArchivesNum 

func GetArchivesNum(gs *GcsUtil, benchmarkArgs, pagesetType string) (int, error)

GetArchivesNum returns the number of archives for the specified pagesetType. -1 is returned if USE_LIVE_SITES_FLAGS is specified or if there is an error.

func GetCASRoot 

func GetCASRoot(local, runOnMaster bool) (string, error)

GetCASRoot returns the location of CT's CAS inputs. local should be set to true when debugging locally. runOnMaster should be set on ctfe. If both are false then it is assumed that we are running on a swarming bot.

func GetChromiumHash 

func GetChromiumHash(ctx context.Context, gitExec string) (string, error)

GetChromiumHash uses ls-remote to find and return Chromium's Tot commit hash.

func GetCipdPackageFromAsset 

func GetCipdPackageFromAsset(assetName string) (string, error)

GetCipdPackageFromAsset returns a string of the format "path:package_name:version". It returns the latest version of the asset via gitiles.

func GetClosedChannelOfPagesets 

func GetClosedChannelOfPagesets(fileInfos []fs.DirEntry) chan string

Returns channel that contains all pageset file names without the timestamp file and pyc files.

func GetCurrentTs 

func GetCurrentTs() string

func GetCurrentTsInt64 

func GetCurrentTsInt64() int64

func GetCustomPages 

func GetCustomPages(customWebPagesFilePath string) ([]string, error)

GetCustomPages returns the specified custom webpages. If Custom webpages are not specified then it returns an empty slice.

func GetCustomPagesWithinRange 

func GetCustomPagesWithinRange(startRange, num int, customWebpages []string) []string

func GetHashesFromBuild 

func GetHashesFromBuild(chromiumBuild string) (string, string)

GetHashesFromBuild returns the Chromium and Skia hashes from a CT build string. Example build string: try-27af50f-d5dcd58-rmistry-20151026102511-nopatch.

func GetIntFlagValue 

func GetIntFlagValue(benchmarkArgs, flagName string, defaultValue int) int

GetIntFlagValue returns the defaultValue if the specified flag name is not in benchmarkArgs.

func GetMaxPagesPerBotValue 

func GetMaxPagesPerBotValue(benchmarkArgs string, defaultValue int) int

GetMaxPagesPerBotValue returns the defaultValue if "--max-pages-per-bot" is not specified in benchmarkArgs. 

func GetMetricsAnalysisOutputLink(runID string) string

func GetNumAnalysisRetriesValue 

func GetNumAnalysisRetriesValue(benchmarkArgs string, defaultValue int) int

GetNumAnalysisRetriesValue returns the defaultValue if "--num-analysis-retries" is not specified in benchmarkArgs.

func GetNumPages 

func GetNumPages(pagesetType, customWebPagesFilePath string) (int, error)

GetNumPages returns the number of specified custom webpages. If Custom webpages are not specified then the number of pages associated with the pageset type is returned.

func GetNumPagesPerBot 

func GetNumPagesPerBot(repeatValue, maxPagesPerBot int) int

GetNumPagesPerBot returns the number of web pages each worker should process.

func GetPatchFromStorage 

func GetPatchFromStorage(patchId string) (string, error)

func GetPathToPyFiles 

func GetPathToPyFiles(local bool) (string, error)

GetPathToPyFiles returns the location of CT's python scripts. local should be set to true if we need the location of py files when debugging locally.

func GetPathToTelemetryBinaries 

func GetPathToTelemetryBinaries(local bool) string

GetPathToTelemetryBinaries returns the location of Telemetry binaries.

func GetPathToTelemetryCTBinaries 

func GetPathToTelemetryCTBinaries(local bool) string

GetPathToTelemetryBinaries returns the location of CT binaries in Telemetry. 

func GetPerfNoPatchOutputLink(runID string) string
func GetPerfOutputLink(runID string) string

func GetPerfOutputLinkBase 

func GetPerfOutputLinkBase(runID string) string

func GetPerfRemoteDir 

func GetPerfRemoteDir(runID string) string

func GetPerfRemoteHTMLDir 

func GetPerfRemoteHTMLDir(runID string) string
func GetPerfWithPatchOutputLink(runID string) string

func GetRankFromPageset 

func GetRankFromPageset(pagesetFileName string) (int, error)

func GetRepeatValue 

func GetRepeatValue(benchmarkArgs string, defaultValue int) int

GetRepeatValue returns the defaultValue if "--pageset-repeat" is not specified in benchmarkArgs.

func GetRowsFromCSV 

func GetRowsFromCSV(csvPath string) ([]string, [][]string, error)

GetRowsFromCSV reads the provided CSV and returns it's headers (first row) and values (all other rows).

func GetRunBenchmarkOutput 

func GetRunBenchmarkOutput(b bytes.Buffer) (string, error)

func GetRunBenchmarkTimeoutValue 

func GetRunBenchmarkTimeoutValue(benchmarkArgs string, defaultValue int) int

GetRunBenchmarkTimeoutValue returns the defaultValue if "--run_benchmark_timeout" is not specified in benchmarkArgs.

func GetStartRange 

func GetStartRange(workerNum, numPagesPerBot int) int

GetStartRange returns the range worker should start processing at based on its num and how many pages it is allowed to process.

func GetStrFlagValue 

func GetStrFlagValue(benchmarkArgs, flagName, defaultValue string) string

GetStrFlagValue returns the defaultValue if the specified flag name is not in benchmarkArgs.

func GetTimeFromTs 

func GetTimeFromTs(formattedTime string) time.Time

func GetUserAgentValue 

func GetUserAgentValue(benchmarkArgs, defaultValue string) string

GetUserAgentValue returns the defaultValue if "--user-agent" is not specified in benchmarkArgs.

func InstallChromeAPK 

func InstallChromeAPK(ctx context.Context, chromiumApkPath string) error

func MakeSwarmingTaskRequest 

func MakeSwarmingTaskRequest(ctx context.Context, taskName, casDigest string, cipdPkgs, cmd, tags []string, dims map[string]string, envPrefixes map[string][]string, priority int32, ioTimeoutSecs time.Duration, casClient cas.CAS) (*apipb.NewTaskRequest, error)

func MergeUploadCSVFiles 

func MergeUploadCSVFiles(ctx context.Context, runID, pathToPyFiles string, gs *GcsUtil, totalPages, maxPagesPerBot int, handleStrings bool, repeatValue int) (string, []string, error)

func MergeUploadCSVFilesOnWorkers 

func MergeUploadCSVFilesOnWorkers(ctx context.Context, localOutputDir, pathToPyFiles, runID, remoteDir, valueColumnName string, gs *GcsUtil, startRange int, handleStrings, addRanks bool, pageRankToAdditionalFields map[string]map[string]string) error

func MkdirAll 

func MkdirAll(name string, perm os.FileMode)

MkdirAll creates the specified path and logs an error if one is returned.

func PatchesAreEmpty 

func PatchesAreEmpty(patches []string) bool

func RemoveFlagsFromArgs 

func RemoveFlagsFromArgs(benchmarkArgs string, flags ...string) string

func Rename 

func Rename(oldpath, newpath string)

Rename renames the specified file and logs an error if one is returned.

func ResetCheckout 

func ResetCheckout(ctx context.Context, dir, resetTo, checkoutArg, gitExec string) error

ResetCheckout resets the specified Git checkout.

func ResetChromiumCheckout 

func ResetChromiumCheckout(ctx context.Context, chromiumSrcDir, gitExec string) error

func RunBenchmark 

func RunBenchmark(ctx context.Context, fileInfoName, pathToPagesets, pathToPyFiles, localOutputDir, chromiumBinary, runID, browserExtraArgs, benchmarkName, targetPlatform, benchmarkExtraArgs, pagesetType string, defaultRepeatValue int, runOnSwarming bool) (string, error)

RunBenchmark runs the specified benchmark with the specified arguments. It prints the output of the run_benchmark command and also returns the output in case the caller needs to do any post-processing on it. In case of any errors the output will be empty.

func SavePatchToStorage 

func SavePatchToStorage(patch string) (string, error)

func SetVarsForLocal 

func SetVarsForLocal()

func SyncDir 

func SyncDir(ctx context.Context, dir string, revisions map[string]string, additionalArgs []string, gitExec string) error

SyncDir runs "git pull" and "gclient sync" on the specified directory. The revisions map enforces revision/hash for the solutions with the format branch@rev.

func TimeTrack 

func TimeTrack(start time.Time, name string)

func TriggerBuildRepoSwarmingTask 

func TriggerBuildRepoSwarmingTask(ctx context.Context, taskName, runID, repoAndTarget, targetPlatform, serviceAccountJSON, gnArgs string, hashes, patches, cipdPkgs []string, singleBuild, local bool, hardTimeout, ioTimeout time.Duration, swarmingClient swarmingv2.SwarmingV2Client, casClient cas.CAS) ([]string, error)

TriggerBuildRepoSwarmingTask triggers a swarming task which runs the build_repo worker script which will return a list of remote build directories.

func TriggerIsolateTelemetrySwarmingTask 

func TriggerIsolateTelemetrySwarmingTask(ctx context.Context, taskName, runID, chromiumHash, serviceAccountJSON, targetPlatform string, patches []string, hardTimeout, ioTimeout time.Duration, local bool, swarmingClient swarmingv2.SwarmingV2Client, casClient cas.CAS) (string, error)

TriggerIsolateTelemetrySwarmingTask triggers a swarming task which runs the isolate_telemetry worker script to upload telemetry to CAS and returns the resulting digest.

func TriggerMasterScriptSwarmingTask 

func TriggerMasterScriptSwarmingTask(ctx context.Context, runID, taskName string, local bool, cmd []string, casSpec *CasSpec, swarmingClient swarmingv2.SwarmingV2Client, casClient cas.CAS) (string, error)

func TriggerSwarmingTask 

func TriggerSwarmingTask(ctx context.Context, pagesetType, taskPrefix, runID, targetPlatform string, casSpec *CasSpec, hardTimeout, ioTimeout time.Duration, priority, maxPagesPerBot, numPages int, runOnGCE, local bool, repeatValue int, baseCmd []string, swarmingClient swarmingv2.SwarmingV2Client, casClient cas.CAS) (int, error)

TriggerSwarmingTask returns the number of triggered tasks and an error (if any).

func UnInstallChromeAPK 

func UnInstallChromeAPK(ctx context.Context)

UnInstallChromeAPK uninstalls the chrome APK from the Android device.

func UploadToCAS 

func UploadToCAS(ctx context.Context, casClient cas.CAS, casSpec *CasSpec, local, runOnMaster bool) (string, error)

UploadToCAS uploads the given CasSpec and returns the resulting digest.

func WritePageset 

func WritePageset(filePath, userAgent, archiveFilePath, url string) error

Types

type CTBenchData 

type CTBenchData struct {
	*format.BenchData
	RunID string `json:"runID"`
}

Extend format.BenchData to include RunID.

type CasSpec 

type CasSpec struct {
	Paths           []string
	IncludeCasSpecs []*CasSpec
	IncludeDigests  []string
}

CasSpec describes a set of files to upload to CAS.

func CasBuildRepoLinux 

func CasBuildRepoLinux() *CasSpec

CasSpecs for build scripts.

func CasCaptureArchives 

func CasCaptureArchives() *CasSpec

func CasCaptureArchivesMaster 

func CasCaptureArchivesMaster() *CasSpec

func CasChromiumAnalysisLinux 

func CasChromiumAnalysisLinux() *CasSpec

func CasChromiumAnalysisMaster 

func CasChromiumAnalysisMaster() *CasSpec

func CasChromiumPerfLinux 

func CasChromiumPerfLinux() *CasSpec

func CasChromiumPerfMaster 

func CasChromiumPerfMaster() *CasSpec

func CasCreatePagesets 

func CasCreatePagesets() *CasSpec

CasSpecs for worker scripts.

func CasCreatePagesetsMaster 

func CasCreatePagesetsMaster() *CasSpec

CasSpecs for master scripts.

func CasIsolateTelemetryLinux 

func CasIsolateTelemetryLinux() *CasSpec

func CasMetricsAnalysis 

func CasMetricsAnalysis() *CasSpec

func CasMetricsAnalysisMaster 

func CasMetricsAnalysisMaster() *CasSpec

func CasPython 

func CasPython() *CasSpec

type GcsUtil 

type GcsUtil struct {
	// contains filtered or unexported fields
}

func NewGcsUtil 

func NewGcsUtil(client *http.Client) (*GcsUtil, error)

NewGcsUtil initializes and returns a utility for CT interations with Google Storage. If client is nil then a client is created with the default token source.

func (*GcsUtil) DeleteRemoteDir 

func (gs *GcsUtil) DeleteRemoteDir(gsDir string) error

func (*GcsUtil) DeleteRemoteDirLogErr 

func (gs *GcsUtil) DeleteRemoteDirLogErr(gsDir string)

DeleteRemoteDirLogErr wraps DeleteRemoteDir and logs an error if one is returned.

func (*GcsUtil) DownloadChromiumBuild 

func (gs *GcsUtil) DownloadChromiumBuild(chromiumBuild, chromiumBinaryName string) (string, error)

DownloadChromiumBuild downloads the specified Chromium build from Google Storage to a local dir. It returns the path to the chrome binary's dir.

func (*GcsUtil) DownloadChromiumBuildFromTo 

func (gs *GcsUtil) DownloadChromiumBuildFromTo(localDir, bucket, gsDir, zipName, chromiumBinaryName string) (string, error)

DownloadChromiumBuildFromTo downloads a chromium build from the specified gs://${bucket}/${gsDir}/${zipName} to the localDir. It returns the path to the chrome binary's dir.

func (*GcsUtil) DownloadRemoteFile 

func (gs *GcsUtil) DownloadRemoteFile(remotePath, localPath string) error

DownloadRemoteFile calls DownloadRemoteFileFromBucket with CT's default bucket.

func (*GcsUtil) DownloadRemoteFileFromBucket 

func (gs *GcsUtil) DownloadRemoteFileFromBucket(bucket, remotePath, localPath string) error

DownloadRemoteFileFromBucket downloads the specified remote path into the specified local file. This function has been tested to download very large files (~33GB). TODO(rmistry): Update all code that downloads remote files to use this or the DownloadRemoteFile method.

func (*GcsUtil) DownloadSwarmingArtifacts 

func (gs *GcsUtil) DownloadSwarmingArtifacts(localDir, remoteDirName, pagesetType string, startRange, num int) (map[string]int, error)

DownloadSwarmingArtifacts downloads the specified artifacts from Google Storage to a local dir. The Google storage directory is assumed to have numerical subdirs Eg: {1..1000}. This function downloads the contents of those directories into a local directory without the numerical subdirs. Returns the ranking/index of the downloaded artifact.

func (*GcsUtil) GetRemoteDirCount 

func (gs *GcsUtil) GetRemoteDirCount(gsDir string) (int, error)

GetRemoteDirCount returns the number of objects in the specified dir.

func (*GcsUtil) GetRemoteFileContents 

func (gs *GcsUtil) GetRemoteFileContents(filePath string) (io.ReadCloser, error)

Returns the response body of the specified GCS file from the default CT bucket. Client must close the response body when finished with it.

func (*GcsUtil) GetRemoteFileContentsFromBucket 

func (gs *GcsUtil) GetRemoteFileContentsFromBucket(bucket, filePath string) (io.ReadCloser, error)

Returns the response body of the specified GCS file. Client must close the response body when finished with it.

func (*GcsUtil) UploadDir 

func (gs *GcsUtil) UploadDir(localDir, gsDir string, cleanDir bool) error

UploadDir uploads the specified local dir into the specified Google Storage dir.

func (*GcsUtil) UploadFile 

func (gs *GcsUtil) UploadFile(fileName, localDir, gsDir string) error

UploadFile calls UploadFileToBucket with CT's default bucket.

func (*GcsUtil) UploadFileToBucket 

func (gs *GcsUtil) UploadFileToBucket(fileName, localDir, gsDir, bucket string) error

UploadFileToBucket uploads the specified file to the remote dir of the bucket in Google Storage. It also sets the appropriate ACLs on the uploaded file.

func (*GcsUtil) UploadSwarmingArtifacts 

func (gs *GcsUtil) UploadSwarmingArtifacts(dirName, pagesetType string) error

UploadSwarmingArtifact uploads the specified local artifacts to Google Storage.

type Pageset 

type Pageset struct {
	UserAgent       string `json:"user_agent"`
	ArchiveDataFile string `json:"archive_data_file"`
	UrlsList        string `json:"urls_list"`
}

type PagesetTypeInfo 

type PagesetTypeInfo struct {
	NumPages                   int
	CSVSource                  string
	UserAgent                  string
	CaptureArchivesTimeoutSecs int
	CreatePagesetsTimeoutSecs  int
	RunChromiumPerfTimeoutSecs int
	Description                string
}

type PagesetVars 

type PagesetVars struct {
	// A comma separated list of URLs.
	UrlsList string `json:"urls_list"`
	// Will be either "mobile" or "desktop".
	UserAgent string `json:"user_agent"`
	// The location of the web page's WPR data file.
	ArchiveDataFile string `json:"archive_data_file"`
}

Contains the data included in CT pagesets.

func ReadPageset 

func ReadPageset(pagesetPath string) (PagesetVars, error)

type TimeoutTracker 

type TimeoutTracker struct {
	// contains filtered or unexported fields
}

func (*TimeoutTracker) Increment 

func (t *TimeoutTracker) Increment()

func (*TimeoutTracker) Read 

func (t *TimeoutTracker) Read() int

func (*TimeoutTracker) Reset 

func (t *TimeoutTracker) Reset()

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.