executions

package
v0.14.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Sep 14, 2020 License: MIT Imports: 15 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	// ScannerConfigActivityName is the activity name ScannerConfigActivity
	ScannerConfigActivityName = "cadence-sys-executions-scanner-config-activity"
	// ScannerScanShardActivityName is the activity name for ScanShardActivity
	ScannerScanShardActivityName = "cadence-sys-executions-scanner-scan-shard-activity"
	// ScannerEmitMetricsActivityName is the activity name for ScannerEmitMetricsActivity
	ScannerEmitMetricsActivityName = "cadence-sys-executions-scanner-emit-metrics-activity"
	// FixerCorruptedKeysActivityName is the activity name for FixerCorruptedKeysActivity
	FixerCorruptedKeysActivityName = "cadence-sys-executions-fixer-corrupted-keys-activity"
	// FixerFixShardActivityName is the activity name for FixShardActivity
	FixerFixShardActivityName = "cadence-sys-executions-fixer-fix-shard-activity"

	// ErrScanWorkflowNotClosed indicates fix was attempted on scan workflow which was not finished
	ErrScanWorkflowNotClosed = "scan workflow is not closed, only can run fix on output of finished scan workflow"
	// ErrSerialization indicates a serialization or deserialization error occurred
	ErrSerialization = "encountered serialization error"
)
View Source 
const (
	// ConcreteScannerContextKey is the key used to access ScannerContext in activities for concrete executions
	ConcreteScannerContextKey = ContextKey(0)
	// ConcreteFixerContextKey is the key used to access FixerContext in activities for concrete executions
	ConcreteFixerContextKey = ContextKey(1)
	// ConcreteScannerContextKey is the key used to access ScannerContext in activities for current executions
	CurrentScannerContextKey = ContextKey(2)
	// ConcreteFixerContextKey is the key used to access FixerContext in activities for current executions
	CurrentFixerContextKey = ContextKey(3)

	// ShardReportQuery is the query name for the query used to get a single shard's report
	ShardReportQuery = "shard_report"
	// ShardStatusQuery is the query name for the query used to get the status of all shards
	ShardStatusQuery = "shard_status"
	// ShardStatusSummaryQuery is the query name for the query used to get the shard status -> counts map
	ShardStatusSummaryQuery = "shard_status_summary"
	// AggregateReportQuery is the query name for the query used to get the aggregate result of all finished shards
	AggregateReportQuery = "aggregate_report"
	// ShardCorruptKeysQuery is the query name for the query used to get all completed shards with at least one corruption
	ShardCorruptKeysQuery = "shard_corrupt_keys"
	// ShardSizeQuery is the query name for the query used to get the number of executions per shard in sorted order
	ShardSizeQuery = "shard_size"

	// ShardStatusRunning indicates the shard has not completed yet
	ShardStatusRunning ShardStatus = "running"
	// ShardStatusSuccess indicates the scan on the shard ran successfully
	ShardStatusSuccess ShardStatus = "success"
	// ShardStatusControlFlowFailure indicates the scan on the shard failed
	ShardStatusControlFlowFailure ShardStatus = "control_flow_failure"
)

Variables

View Source 
var ScanTypeFixerContextKeyMap = map[common.ScanType]interface{}{
	common.ConcreteExecutionType: ConcreteFixerContextKey,
	common.CurrentExecutionType:  CurrentFixerContextKey,
}
View Source 
var ScanTypeScannerContextKeyMap = map[common.ScanType]interface{}{
	common.ConcreteExecutionType: ConcreteScannerContextKey,
	common.CurrentExecutionType:  CurrentScannerContextKey,
}

Functions

func FixShardActivity added in v0.13.0 

func FixShardActivity(
	activityCtx context.Context,
	params FixShardActivityParams,
) ([]common.ShardFixReport, error)

FixShardActivity will fix a collection of shards.

func FixerWorkflow added in v0.13.0 

func FixerWorkflow(
	ctx workflow.Context,
	params FixerWorkflowParams,
) error

FixerWorkflow is the workflow that fixes all concrete executions from a scan output.

func ScanShardActivity added in v0.13.0 

func ScanShardActivity(
	activityCtx context.Context,
	params ScanShardActivityParams,
) ([]common.ShardScanReport, error)

ScanShardActivity will scan a collection of shards for invariant violations.

func ScannerEmitMetricsActivity added in v0.13.0 

func ScannerEmitMetricsActivity(
	activityCtx context.Context,
	params ScannerEmitMetricsActivityParams,
) error

ScannerEmitMetricsActivity will emit metrics for a complete run of scanner

func ScannerWorkflow added in v0.13.0 

func ScannerWorkflow(
	ctx workflow.Context,
	params ScannerWorkflowParams,
) error

ScannerWorkflow is the workflow that scans over all executions for the given scan type

Types

type AggregateFixReportResult added in v0.13.0 

type AggregateFixReportResult common.ShardFixStats

AggregateFixReportResult indicates the result of summing together all shard reports that have finished for fix.

type AggregateScanReportResult added in v0.13.0 

type AggregateScanReportResult common.ShardScanStats

AggregateScanReportResult indicates the result of summing together all shard reports which have finished scan.

type ContextKey added in v0.13.0 

type ContextKey int

ContextKey is the type which identifies context keys

type CorruptedKeysEntry added in v0.13.0 

type CorruptedKeysEntry struct {
	ShardID       int
	CorruptedKeys common.Keys
}

CorruptedKeysEntry is a pair of shardID and corrupted keys

type DynamicConfigInvariantCollections added in v0.13.0 

type DynamicConfigInvariantCollections struct {
	InvariantCollectionMutableState dynamicconfig.BoolPropertyFn
	InvariantCollectionHistory      dynamicconfig.BoolPropertyFn
}

DynamicConfigInvariantCollections is the portion of ScannerWorkflowDynamicConfig which indicates which collections of invariants should be run

type FixReportError added in v0.13.0 

type FixReportError struct {
	Reports  []common.ShardFixReport
	ErrorStr *string
}

FixReportError is a type that is used to send either error or report on a channel. Exactly one of Report and ErrorStr should be non-nil.

type FixShardActivityParams added in v0.13.0 

type FixShardActivityParams struct {
	CorruptedKeysEntries        []CorruptedKeysEntry
	ResolvedFixerWorkflowConfig ResolvedFixerWorkflowConfig
	ScanType                    common.ScanType
}

FixShardActivityParams is the parameter for FixShardActivity

type FixShardHeartbeatDetails added in v0.13.0 

type FixShardHeartbeatDetails struct {
	LastShardIndexHandled int
	Reports               []common.ShardFixReport
}

FixShardHeartbeatDetails is the heartbeat details for the fix shard

type FixerContext added in v0.13.0 

type FixerContext struct {
	Resource resource.Resource
	Scope    metrics.Scope
}

FixerContext is the resource that is available to activities under ConcreteFixerContextKey

type FixerCorruptedKeysActivityParams added in v0.13.0 

type FixerCorruptedKeysActivityParams struct {
	ScannerWorkflowWorkflowID string
	ScannerWorkflowRunID      string
	StartingShardID           *int
	ScanType                  common.ScanType
}

FixerCorruptedKeysActivityParams is the parameter for FixerCorruptedKeysActivity

type FixerCorruptedKeysActivityResult added in v0.13.0 

type FixerCorruptedKeysActivityResult struct {
	CorruptedKeys             []CorruptedKeysEntry
	MinShard                  *int
	MaxShard                  *int
	ShardQueryPaginationToken ShardQueryPaginationToken
}

FixerCorruptedKeysActivityResult is the result of FixerCorruptedKeysActivity

func FixerCorruptedKeysActivity added in v0.13.0 

func FixerCorruptedKeysActivity(
	activityCtx context.Context,
	params FixerCorruptedKeysActivityParams,
) (*FixerCorruptedKeysActivityResult, error)

FixerCorruptedKeysActivity will fetch the keys of blobs from shards with corruptions from a completed scan workflow. If scan workflow is not closed or if query fails activity will return an error. Accepts as input the shard to start query at and returns a next page token, therefore this activity can be used to do pagination.

type FixerWorkflowConfigOverwrites added in v0.13.0 

type FixerWorkflowConfigOverwrites struct {
	Concurrency             *int
	BlobstoreFlushThreshold *int
	ActivityBatchSize       *int
	InvariantCollections    *InvariantCollections
}

FixerWorkflowConfigOverwrites enables overwriting the default values. If provided workflow will favor overwrites over defaults. Any overwrites that are left as nil will fall back to defaults.

type FixerWorkflowParams added in v0.13.0 

type FixerWorkflowParams struct {
	ScannerWorkflowWorkflowID     string
	ScannerWorkflowRunID          string
	FixerWorkflowConfigOverwrites FixerWorkflowConfigOverwrites
	ScanType                      common.ScanType
}

FixerWorkflowParams are the parameters to the fix workflow

type InvariantCollections added in v0.13.0 

type InvariantCollections struct {
	InvariantCollectionMutableState bool
	InvariantCollectionHistory      bool
}

InvariantCollections represents the resolved set of invariant collections that scanner workflow should run

type PaginatedShardQueryRequest added in v0.13.0 

type PaginatedShardQueryRequest struct {
	// StartingShardID is the first shard to start iteration from.
	// Setting to nil will start iteration from the beginning of the shards.
	StartingShardID *int
	// LimitShards indicates the maximum number of results that can be returned.
	// If nil or larger than allowed maximum, will default to maximum allowed.
	LimitShards *int
}

PaginatedShardQueryRequest is the request used for queries which return results over all shards

type ResolvedFixerWorkflowConfig added in v0.13.0 

type ResolvedFixerWorkflowConfig struct {
	Concurrency             int
	BlobstoreFlushThreshold int
	ActivityBatchSize       int
	InvariantCollections    InvariantCollections
}

ResolvedFixerWorkflowConfig is the resolved config after reading defaults and applying overwrites.

type ResolvedScannerWorkflowConfig added in v0.13.0 

type ResolvedScannerWorkflowConfig struct {
	Enabled                 bool
	Concurrency             int
	ExecutionsPageSize      int
	BlobstoreFlushThreshold int
	ActivityBatchSize       int
	InvariantCollections    InvariantCollections
}

ResolvedScannerWorkflowConfig is the resolved config after reading dynamic config and applying overwrites.

func ScannerConfigActivity added in v0.13.0 

func ScannerConfigActivity(
	activityCtx context.Context,
	params ScannerConfigActivityParams,
) (ResolvedScannerWorkflowConfig, error)

ScannerConfigActivity will read dynamic config, apply overwrites and return a resolved config.

type ScanReportError added in v0.13.0 

type ScanReportError struct {
	Reports  []common.ShardScanReport
	ErrorStr *string
}

ScanReportError is a type that is used to send either error or report on a channel. Exactly one of Report and ErrorStr should be non-nil.

type ScanShardActivityParams added in v0.13.0 

type ScanShardActivityParams struct {
	Shards                  []int
	ExecutionsPageSize      int
	BlobstoreFlushThreshold int
	InvariantCollections    InvariantCollections
	ScanType                common.ScanType
}

ScanShardActivityParams is the parameter for ScanShardActivity

type ScanShardHeartbeatDetails added in v0.13.0 

type ScanShardHeartbeatDetails struct {
	LastShardIndexHandled int
	Reports               []common.ShardScanReport
}

ScanShardHeartbeatDetails is the heartbeat details for scan shard

type ScannerConfigActivityParams added in v0.13.0 

type ScannerConfigActivityParams struct {
	Overwrites ScannerWorkflowConfigOverwrites
	ScanType   common.ScanType
}

ScannerConfigActivityParams is the parameter for ScannerConfigActivity

type ScannerContext added in v0.13.0 

type ScannerContext struct {
	Resource                     resource.Resource
	Scope                        metrics.Scope
	ScannerWorkflowDynamicConfig *ScannerWorkflowDynamicConfig
}

ScannerContext is the resource that is available in activities under ConcreteScannerContextKey context key

type ScannerEmitMetricsActivityParams added in v0.13.0 

type ScannerEmitMetricsActivityParams struct {
	ShardSuccessCount            int
	ShardControlFlowFailureCount int
	AggregateReportResult        AggregateScanReportResult
	ShardDistributionStats       ShardDistributionStats
	ScanType                     common.ScanType
}

ScannerEmitMetricsActivityParams is the parameter for ScannerEmitMetricsActivity

type ScannerWorkflowConfigOverwrites added in v0.13.0 

type ScannerWorkflowConfigOverwrites struct {
	Enabled                 *bool
	Concurrency             *int
	ExecutionsPageSize      *int
	BlobstoreFlushThreshold *int
	ActivityBatchSize       *int
	InvariantCollections    *InvariantCollections
}

ScannerWorkflowConfigOverwrites enables overwriting the values in dynamic config. If provided workflow will favor overwrites over dynamic config. Any overwrites that are left as nil will fall back to using dynamic config.

type ScannerWorkflowDynamicConfig added in v0.13.0 

type ScannerWorkflowDynamicConfig struct {
	Enabled                           dynamicconfig.BoolPropertyFn
	Concurrency                       dynamicconfig.IntPropertyFn
	ExecutionsPageSize                dynamicconfig.IntPropertyFn
	BlobstoreFlushThreshold           dynamicconfig.IntPropertyFn
	ActivityBatchSize                 dynamicconfig.IntPropertyFn
	DynamicConfigInvariantCollections DynamicConfigInvariantCollections
}

ScannerWorkflowDynamicConfig is the dynamic config for scanner workflow

type ScannerWorkflowParams 

type ScannerWorkflowParams struct {
	Shards                          Shards
	ScannerWorkflowConfigOverwrites ScannerWorkflowConfigOverwrites
	ScanType                        common.ScanType
}

ScannerWorkflowParams are the parameters to the scan workflow

type ShardCorruptKeysQueryResult added in v0.13.0 

type ShardCorruptKeysQueryResult struct {
	Result                    ShardCorruptKeysResult
	ShardQueryPaginationToken ShardQueryPaginationToken
}

ShardCorruptKeysQueryResult is the query result for ShardCorruptKeysQuery

type ShardCorruptKeysResult added in v0.13.0 

type ShardCorruptKeysResult map[int]common.Keys

ShardCorruptKeysResult is a map of all shards which have finished scan successfully and have at least one corruption

type ShardDistributionStats added in v0.14.0 

type ShardDistributionStats struct {
	Max    int64
	Median int64
	Min    int64
	P90    int64
	P75    int64
	P25    int64
	P10    int64
}

ShardDistributionStats contains stats on the distribution of executions in shards. It is used by the ScannerEmitMetricsActivityParams.

type ShardQueryPaginationToken added in v0.13.0 

type ShardQueryPaginationToken struct {
	// NextShardID is one greater than the highest shard returned in the current query.
	// NextShardID is nil if IsDone is true.
	// It is possible to get NextShardID != nil and on the next call to get an empty result with IsDone = true.
	NextShardID *int
	IsDone      bool
}

ShardQueryPaginationToken is used to return information used to make the next query

type ShardRange added in v0.13.0 

type ShardRange struct {
	Min int
	Max int
}

ShardRange identifies a set of shards based on min (inclusive) and max (exclusive)

type ShardSizeQueryRequest added in v0.14.0 

type ShardSizeQueryRequest struct {
	StartIndex int
	EndIndex   int
}

ShardSizeQueryRequest is the request used for ShardSizeQuery. The following must be true: 0 <= StartIndex < EndIndex <= len(shards successfully finished) The following must be true: EndIndex - StartIndex <= maxShardQueryResult. StartIndex is inclusive, EndIndex is exclusive.

type ShardSizeQueryResult added in v0.14.0 

type ShardSizeQueryResult []ShardSizeTuple

ShardSizeQueryResult is the result from ShardSizeQuery. Contains sorted list of shards, sorted by the number of executions per shard.

type ShardSizeTuple added in v0.14.0 

type ShardSizeTuple struct {
	ShardID         int
	ExecutionsCount int64
}

ShardSizeTuple indicates the size and sorted index of a single shard

type ShardStatus added in v0.13.0 

type ShardStatus string

ShardStatus is the type which indicates the status of a shard scan.

type ShardStatusQueryResult added in v0.13.0 

type ShardStatusQueryResult struct {
	Result                    ShardStatusResult
	ShardQueryPaginationToken ShardQueryPaginationToken
}

ShardStatusQueryResult is the query result for ShardStatusQuery

type ShardStatusResult added in v0.13.0 

type ShardStatusResult map[int]ShardStatus

ShardStatusResult indicates the status for all shards

type ShardStatusSummaryResult added in v0.14.0 

type ShardStatusSummaryResult map[ShardStatus]int

ShardStatusSummaryResult indicates the counts of shards in each status

type Shards added in v0.13.0 

type Shards struct {
	List  []int
	Range *ShardRange
}

Shards identify the shards that should be scanned. Exactly one of List or Range should be non-nil.

func (Shards) Flatten added in v0.14.0 

func (s Shards) Flatten() ([]int, int, int)

Flatten flattens Shards to a list of shard IDs and finds the min/max shardID

func (Shards) Validate added in v0.14.0 

func (s Shards) Validate() error

Validate validates shard list or range

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.