Documentation ¶
Overview ¶
Package integration implements an integration test framework for Pulumi. This framework supports running the full resource lifecycle in all supported languages, and then performing assertions on the state (resources, events, errors, etc.)
Index ¶
- Constants
- Variables
- func AssertHTTPResultWithRetry(t *testing.T, output interface{}, headers map[string]string, ...) bool
- func CopyDir(src, dst string) error
- func CopyFile(src, dst string) error
- func CreateBasicPulumiRepo(e *testing.Environment)
- func CreatePulumiRepo(e *testing.Environment, projectFileContent string)
- func DecodeMapString(val string) (map[string]string, error)
- func GetLogs(t *testing.T, provider, region string, stackInfo RuntimeValidationStackInfo, ...) *[]operations.LogEntry
- func GetStacks(e *testing.Environment) ([]string, *string)
- func GoPath() (string, error)
- func MakeTempBackend(t *testing.T) string
- func ProgramTest(t *testing.T, opts *ProgramTestOptions)
- func ReplaceInFile(old, new, path string) error
- func RunCommand(t *testing.T, name string, args []string, wd string, opts *ProgramTestOptions) error
- func RunCommandPulumiHome(t *testing.T, name string, args []string, wd string, opts *ProgramTestOptions, ...) error
- type ConfigValue
- type EditDir
- type Environment
- type LocalDependency
- type ProgramTestOptions
- func (opts *ProgramTestOptions) GetDebugLogLevel() int
- func (opts *ProgramTestOptions) GetDebugUpdates() bool
- func (opts *ProgramTestOptions) GetStackName() tokens.QName
- func (opts *ProgramTestOptions) GetStackNameWithOwner() tokens.QName
- func (opts *ProgramTestOptions) GetUseSharedVirtualEnv() bool
- func (opts ProgramTestOptions) With(overrides ProgramTestOptions) ProgramTestOptions
- type ProgramTester
- func (pt *ProgramTester) PreviewAndUpdate(dir string, name string, shouldFail, expectNopPreview, expectNopUpdate bool) error
- func (pt *ProgramTester) RunPulumiCommand(name string, args ...string) error
- func (pt *ProgramTester) TestCleanUp()
- func (pt *ProgramTester) TestLifeCycleDestroy() error
- func (pt *ProgramTester) TestLifeCycleInitAndDestroy() error
- func (pt *ProgramTester) TestLifeCycleInitialize() error
- func (pt *ProgramTester) TestLifeCyclePrepare() error
- func (pt *ProgramTester) TestPreviewUpdateAndEdits() error
- type RuntimeValidationStackInfo
- type S3Reporter
- type TestCommandStats
- type TestStatsReporter
Constants ¶
const ( PythonRuntime = "python" NodeJSRuntime = "nodejs" GoRuntime = "go" DotNetRuntime = "dotnet" YAMLRuntime = "yaml" JavaRuntime = "java" )
Variables ¶
var ErrTestFailed = errors.New("test failed")
Functions ¶
func AssertHTTPResultWithRetry ¶ added in v3.18.0
func AssertHTTPResultWithRetry( t *testing.T, output interface{}, headers map[string]string, maxWait time.Duration, check func(string) bool, ) bool
AssertHTTPResultWithRetry attempts to assert that an HTTP endpoint exists and evaluate its response.
func CopyDir ¶
CopyDir copies a whole directory recursively From https://blog.depado.eu/post/copy-files-and-directories-in-go
func CopyFile ¶
CopyFile copies a single file from src to dst From https://blog.depado.eu/post/copy-files-and-directories-in-go
func CreateBasicPulumiRepo ¶
func CreateBasicPulumiRepo(e *testing.Environment)
CreateBasicPulumiRepo will initialize the environment with a basic Pulumi repository and project file definition. Returns the repo owner and name used.
func CreatePulumiRepo ¶ added in v3.44.0
func CreatePulumiRepo(e *testing.Environment, projectFileContent string)
CreatePulumiRepo will initialize the environment with a basic Pulumi repository and project file definition based on the project file content. Returns the repo owner and name used.
func DecodeMapString ¶
DecodeMapString takes a string of the form key1=value1:key2=value2 and returns a go map.
func GetLogs ¶
func GetLogs( t *testing.T, provider, region string, stackInfo RuntimeValidationStackInfo, query operations.LogQuery, ) *[]operations.LogEntry
GetLogs retrieves the logs for a given stack in a particular region making the query provided.
[provider] should be one of "aws" or "azure"
func GetStacks ¶
func GetStacks(e *testing.Environment) ([]string, *string)
GetStacks returns the list of stacks and current stack by scraping `pulumi stack ls`. Assumes .pulumi is in the current working directory. Fails the test on IO errors.
func MakeTempBackend ¶ added in v3.40.1
MakeTempBackend creates a temporary backend directory which will clean up on test exit.
func ProgramTest ¶
func ProgramTest(t *testing.T, opts *ProgramTestOptions)
ProgramTest runs a lifecycle of Pulumi commands in a program working directory, using the `pulumi` and `yarn` binaries available on PATH. It essentially executes the following workflow:
yarn install yarn link <each opts.Depencies> (+) yarn run build pulumi init (*) pulumi login pulumi stack init integrationtesting pulumi config set <each opts.Config> pulumi config set --secret <each opts.Secrets> pulumi preview pulumi up pulumi stack export --file stack.json pulumi stack import --file stack.json pulumi preview (expected to be empty) pulumi up (expected to be empty) pulumi destroy --yes pulumi stack rm --yes integrationtesting (*) Only if PULUMI_ACCESS_TOKEN is set. (+) Only if `opts.RunBuild` is true.
All commands must return success return codes for the test to succeed, unless ExpectFailure is true.
func ReplaceInFile ¶
ReplaceInFile does a find and replace for a given string within a file.
func RunCommand ¶
func RunCommandPulumiHome ¶ added in v3.109.0
func RunCommandPulumiHome( t *testing.T, name string, args []string, wd string, opts *ProgramTestOptions, pulumiHome string, ) error
RunCommandPulumiHome executes the specified command and additional arguments, wrapping any output in the specialized test output streams that list the location the test is running in, and sets the PULUMI_HOME environment variable.
Types ¶
type ConfigValue ¶
type ConfigValue struct { // The config key to pass to `pulumi config`. Key string // The config value to pass to `pulumi config`. Value string // Secret indicates that the `--secret` flag should be specified when calling `pulumi config`. Secret bool // Path indicates that the `--path` flag should be specified when calling `pulumi config`. Path bool }
ConfigValue is used to provide config values to a test program.
type EditDir ¶
type EditDir struct { Dir string ExtraRuntimeValidation func(t *testing.T, stack RuntimeValidationStackInfo) // Additive is true if Dir should be copied *on top* of the test directory. // Otherwise Dir *replaces* the test directory, except we keep .pulumi/ and Pulumi.yaml and Pulumi.<stack>.yaml. Additive bool // ExpectFailure is true if we expect this test to fail. This is very coarse grained, and will essentially // tolerate *any* failure in the program (IDEA: in the future, offer a way to narrow this down more). ExpectFailure bool // ExpectNoChanges is true if the edit is expected to not propose any changes. ExpectNoChanges bool // Stdout is the writer to use for all stdout messages. Stdout io.Writer // Stderr is the writer to use for all stderr messages. Stderr io.Writer // Verbose may be set to true to print messages as they occur, rather than buffering and showing upon failure. Verbose bool // Run program directory in query mode. QueryMode bool }
EditDir is an optional edit to apply to the example, as subsequent deployments.
type Environment ¶ added in v3.89.0
type Environment struct { // The name of the environment. Name string // The definition of the environment. Definition map[string]any }
Environment is used to create environments for use by test programs.
type LocalDependency ¶ added in v3.38.0
type ProgramTestOptions ¶
type ProgramTestOptions struct { // Dir is the program directory to test. Dir string // Array of NPM packages which must be `yarn linked` (e.g. {"pulumi", "@pulumi/aws"}) Dependencies []string // Map of package names to versions. The test will use the specified versions of these packages instead of what // is declared in `package.json`. Overrides map[string]string // Automatically use the latest dev version of pulumi SDKs if available. InstallDevReleases bool // List of environments to create in order. CreateEnvironments []Environment // List of environments to use. Environments []string // Map of config keys and values to set (e.g. {"aws:region": "us-east-2"}). Config map[string]string // Map of secure config keys and values to set (e.g. {"aws:region": "us-east-2"}). Secrets map[string]string // List of config keys and values to set in order, including Secret and Path options. OrderedConfig []ConfigValue // SecretsProvider is the optional custom secrets provider to use instead of the default. SecretsProvider string // EditDirs is an optional list of edits to apply to the example, as subsequent deployments. EditDirs []EditDir // ExtraRuntimeValidation is an optional callback for additional validation, called before applying edits. ExtraRuntimeValidation func(t *testing.T, stack RuntimeValidationStackInfo) // RelativeWorkDir is an optional path relative to `Dir` which should be used as working directory during tests. RelativeWorkDir string // AllowEmptyPreviewChanges is true if we expect that this test's no-op preview may propose changes (e.g. // because the test is sensitive to the exact contents of its working directory and those contents change // incidentally between the initial update and the empty update). AllowEmptyPreviewChanges bool // AllowEmptyUpdateChanges is true if we expect that this test's no-op update may perform changes (e.g. // because the test is sensitive to the exact contents of its working directory and those contents change // incidentally between the initial update and the empty update). AllowEmptyUpdateChanges bool // ExpectFailure is true if we expect this test to fail. This is very coarse grained, and will essentially // tolerate *any* failure in the program (IDEA: in the future, offer a way to narrow this down more). ExpectFailure bool // ExpectRefreshChanges may be set to true if a test is expected to have changes yielded by an immediate refresh. // This could occur, for example, is a resource's state is constantly changing outside of Pulumi (e.g., timestamps). ExpectRefreshChanges bool // RetryFailedSteps indicates that failed updates, refreshes, and destroys should be retried after a brief // intermission. A maximum of 3 retries will be attempted. RetryFailedSteps bool // SkipRefresh indicates that the refresh step should be skipped entirely. SkipRefresh bool // Require a preview after refresh to be a no-op (expect no changes). Has no effect if SkipRefresh is true. RequireEmptyPreviewAfterRefresh bool // SkipPreview indicates that the preview step should be skipped entirely. SkipPreview bool // SkipUpdate indicates that the update step should be skipped entirely. SkipUpdate bool // SkipExportImport skips testing that exporting and importing the stack works properly. SkipExportImport bool // SkipEmptyPreviewUpdate skips the no-change preview/update that is performed that validates // that no changes happen. SkipEmptyPreviewUpdate bool // SkipStackRemoval indicates that the stack should not be removed. (And so the test's results could be inspected // in the Pulumi Service after the test has completed.) SkipStackRemoval bool // Destroy on cleanup defers stack destruction until the test cleanup step, rather than after // program test execution. This is useful for more realistic stack reference testing, allowing one // project and stack to be stood up and a second to be run before the first is destroyed. // // Implies NoParallel because we expect that another caller to ProgramTest will set that DestroyOnCleanup bool // DestroyExcludeProtected indicates that when the test stack is destroyed, // protected resources should be excluded from the destroy operation. DestroyExcludeProtected bool // Quick implies SkipPreview, SkipExportImport and SkipEmptyPreviewUpdate Quick bool // RequireService indicates that the test must be run against the Pulumi Service RequireService bool // PreviewCommandlineFlags specifies flags to add to the `pulumi preview` command line (e.g. "--color=raw") PreviewCommandlineFlags []string // UpdateCommandlineFlags specifies flags to add to the `pulumi up` command line (e.g. "--color=raw") UpdateCommandlineFlags []string // QueryCommandlineFlags specifies flags to add to the `pulumi query` command line (e.g. "--color=raw") QueryCommandlineFlags []string // RunBuild indicates that the build step should be run (e.g. run `yarn build` for `nodejs` programs) RunBuild bool // RunUpdateTest will ensure that updates to the package version can test for spurious diffs RunUpdateTest bool // DecryptSecretsInOutput will ensure that stack output is passed `--show-secrets` parameter // Used in conjunction with ExtraRuntimeValidation DecryptSecretsInOutput bool // CloudURL is an optional URL to override the default Pulumi Service API (https://api.pulumi-staging.io). The // PULUMI_ACCESS_TOKEN environment variable must also be set to a valid access token for the target cloud. CloudURL string // StackName allows the stack name to be explicitly provided instead of computed from the // environment during tests. StackName string // If non-empty, specifies the value of the `--tracing` flag to pass // to Pulumi CLI, which may be a Zipkin endpoint or a // `file:./local.trace` style url for AppDash tracing. // // Template `{command}` syntax will be expanded to the current // command name such as `pulumi-stack-rm`. This is useful for // file-based tracing since `ProgramTest` performs multiple // CLI invocations that can inadvertently overwrite the trace // file. Tracing string // NoParallel will opt the test out of being ran in parallel. NoParallel bool // PrePulumiCommand specifies a callback that will be executed before each `pulumi` invocation. This callback may // optionally return another callback to be invoked after the `pulumi` invocation completes. PrePulumiCommand func(verb string) (func(err error) error, error) // ReportStats optionally specifies how to report results from the test for external collection. ReportStats TestStatsReporter // Stdout is the writer to use for all stdout messages. Stdout io.Writer // Stderr is the writer to use for all stderr messages. Stderr io.Writer // Verbose may be set to true to print messages as they occur, rather than buffering and showing upon failure. Verbose bool // DebugLogging may be set to anything >0 to enable excessively verbose debug logging from `pulumi`. This // is equivalent to `--logflow --logtostderr -v=N`, where N is the value of DebugLogLevel. This may also // be enabled by setting the environment variable PULUMI_TEST_DEBUG_LOG_LEVEL. DebugLogLevel int // DebugUpdates may be set to true to enable debug logging from `pulumi preview`, `pulumi up`, and // `pulumi destroy`. This may also be enabled by setting the environment variable PULUMI_TEST_DEBUG_UPDATES. DebugUpdates bool // Bin is a location of a `pulumi` executable to be run. Taken from the $PATH if missing. Bin string // YarnBin is a location of a `yarn` executable to be run. Taken from the $PATH if missing. YarnBin string // GoBin is a location of a `go` executable to be run. Taken from the $PATH if missing. GoBin string // PythonBin is a location of a `python` executable to be run. Taken from the $PATH if missing. PythonBin string // PipenvBin is a location of a `pipenv` executable to run. Taken from the $PATH if missing. PipenvBin string // DotNetBin is a location of a `dotnet` executable to be run. Taken from the $PATH if missing. DotNetBin string // Additional environment variables to pass for each command we run. Env []string // Automatically create and use a virtual environment, rather than using the Pipenv tool. This is now the default // behavior, so this option no longer has any affect. To go back to the old behavior use the `UsePipenv` option. UseAutomaticVirtualEnv bool // Use the Pipenv tool to manage the virtual environment. UsePipenv bool UseSharedVirtualEnv *bool SharedVirtualEnvPath string // If set, this hook is called after the `pulumi preview` command has completed. PreviewCompletedHook func(dir string) error // JSONOutput indicates that the `--json` flag should be passed to `up`, `preview`, // `refresh` and `destroy` commands. JSONOutput bool // If set, this hook is called after `pulumi stack export` on the exported file. If `SkipExportImport` is set, this // hook is ignored. ExportStateValidator func(t *testing.T, stack []byte) // If not nil, specifies the logic of preparing a project by // ensuring dependencies. If left as nil, runs default // preparation logic by dispatching on whether the project // uses Node, Python, .NET or Go. PrepareProject func(*engine.Projinfo) error // If not nil, will be run after the project has been prepared. PostPrepareProject func(*engine.Projinfo) error // Array of provider plugin dependencies which come from local packages. LocalProviders []LocalDependency // contains filtered or unexported fields }
ProgramTestOptions provides options for ProgramTest
func (*ProgramTestOptions) GetDebugLogLevel ¶
func (opts *ProgramTestOptions) GetDebugLogLevel() int
func (*ProgramTestOptions) GetDebugUpdates ¶
func (opts *ProgramTestOptions) GetDebugUpdates() bool
func (*ProgramTestOptions) GetStackName ¶
func (opts *ProgramTestOptions) GetStackName() tokens.QName
GetStackName returns a stack name to use for this test.
func (*ProgramTestOptions) GetStackNameWithOwner ¶
func (opts *ProgramTestOptions) GetStackNameWithOwner() tokens.QName
GetStackNameWithOwner gets the name of the stack prepended with an owner, if PULUMI_TEST_OWNER is set. We use this in CI to create test stacks in an organization that all developers have access to, for debugging.
func (*ProgramTestOptions) GetUseSharedVirtualEnv ¶ added in v3.49.0
func (opts *ProgramTestOptions) GetUseSharedVirtualEnv() bool
func (ProgramTestOptions) With ¶
func (opts ProgramTestOptions) With(overrides ProgramTestOptions) ProgramTestOptions
With combines a source set of options with a set of overrides.
type ProgramTester ¶
type ProgramTester struct { TestFinished bool // whether or not the test if finished // contains filtered or unexported fields }
ProgramTester contains state associated with running a single test pass.
func ProgramTestManualLifeCycle ¶
func ProgramTestManualLifeCycle(t *testing.T, opts *ProgramTestOptions) *ProgramTester
ProgramTestManualLifeCycle returns a ProgramTester than must be manually controlled in terms of its lifecycle
func (*ProgramTester) PreviewAndUpdate ¶
func (pt *ProgramTester) PreviewAndUpdate(dir string, name string, shouldFail, expectNopPreview, expectNopUpdate bool, ) error
PreviewAndUpdate runs pulumi preview followed by pulumi up
func (*ProgramTester) RunPulumiCommand ¶ added in v3.72.2
func (pt *ProgramTester) RunPulumiCommand(name string, args ...string) error
RunPulumiCommand runs a Pulumi command in the project directory. For example:
pt.RunPulumiCommand("preview", "--stack", "dev")
func (*ProgramTester) TestCleanUp ¶
func (pt *ProgramTester) TestCleanUp()
TestCleanUp cleans up the temporary directory that a test used
func (*ProgramTester) TestLifeCycleDestroy ¶
func (pt *ProgramTester) TestLifeCycleDestroy() error
TestLifeCycleDestroy destroys a stack and removes it
func (*ProgramTester) TestLifeCycleInitAndDestroy ¶
func (pt *ProgramTester) TestLifeCycleInitAndDestroy() error
TestLifeCycleInitAndDestroy executes the test and cleans up
func (*ProgramTester) TestLifeCycleInitialize ¶
func (pt *ProgramTester) TestLifeCycleInitialize() error
TestLifeCycleInitialize initializes the project directory and stack along with any configuration
func (*ProgramTester) TestLifeCyclePrepare ¶
func (pt *ProgramTester) TestLifeCyclePrepare() error
TestLifeCyclePrepare prepares a test by creating a temporary directory
func (*ProgramTester) TestPreviewUpdateAndEdits ¶
func (pt *ProgramTester) TestPreviewUpdateAndEdits() error
TestPreviewUpdateAndEdits runs the preview, update, and any relevant edits
type RuntimeValidationStackInfo ¶
type RuntimeValidationStackInfo struct { StackName tokens.QName Deployment *apitype.DeploymentV3 RootResource apitype.ResourceV3 Outputs map[string]interface{} Events []apitype.EngineEvent }
RuntimeValidationStackInfo contains details related to the stack that runtime validation logic may want to use.
type S3Reporter ¶
type S3Reporter struct {
// contains filtered or unexported fields
}
S3Reporter is a TestStatsReporter that publises test data to S3
func NewS3Reporter ¶
func NewS3Reporter(region string, bucket string, keyPrefix string) *S3Reporter
NewS3Reporter creates a new S3Reporter that puts test results in the given bucket using the keyPrefix.
func (*S3Reporter) ReportCommand ¶
func (r *S3Reporter) ReportCommand(stats TestCommandStats)
ReportCommand uploads the results of running a command to S3
type TestCommandStats ¶
type TestCommandStats struct { // StartTime is the time at which the command was started StartTime string `json:"startTime"` // EndTime is the time at which the command exited EndTime string `json:"endTime"` // ElapsedSeconds is the time at which the command exited ElapsedSeconds float64 `json:"elapsedSeconds"` // StackName is the name of the stack StackName string `json:"stackName"` // TestId is the unique ID of the test run TestID string `json:"testId"` // StepName is the command line which was invoked StepName string `json:"stepName"` // CommandLine is the command line which was invoked CommandLine string `json:"commandLine"` // TestName is the name of the directory in which the test was executed TestName string `json:"testName"` // IsError is true if the command failed IsError bool `json:"isError"` // The Cloud that the test was run against, or empty for local deployments CloudURL string `json:"cloudURL"` }
TestCommandStats is a collection of data related to running a single command during a test.
type TestStatsReporter ¶
type TestStatsReporter interface {
ReportCommand(stats TestCommandStats)
}
TestStatsReporter reports results and metadata from a test run.