storage

package
v1.47.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jun 5, 2024 License: Apache-2.0 Imports: 23 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

func DataSourceDbfsFile 

func DataSourceDbfsFile() common.Resource

func DataSourceDbfsFilePaths 

func DataSourceDbfsFilePaths() common.Resource

func DbfsFileV0 

func DbfsFileV0() cty.Type

DbfsFileV0 contains v0.2.x schema

func GetOrCreateMountingClusterWithGcpServiceAccount 

func GetOrCreateMountingClusterWithGcpServiceAccount(
	clustersAPI clusters.ClustersAPI, serviceAccount string) (i clusters.ClusterInfo, err error)

GetOrCreateMountingClusterWithGcpServiceAccount ...

func GetOrCreateMountingClusterWithInstanceProfile 

func GetOrCreateMountingClusterWithInstanceProfile(
	clustersAPI clusters.ClustersAPI, instanceProfile string) (i clusters.ClusterInfo, err error)

GetOrCreateMountingClusterWithInstanceProfile ...

func ResourceAWSS3Mount 

func ResourceAWSS3Mount() common.Resource

ResourceAWSS3Mount ...

func ResourceAzureAdlsGen1Mount 

func ResourceAzureAdlsGen1Mount() common.Resource

ResourceAzureAdlsGen1Mount creates the resource

func ResourceAzureAdlsGen2Mount 

func ResourceAzureAdlsGen2Mount() common.Resource

ResourceAzureAdlsGen2Mount creates the resource

func ResourceAzureBlobMount 

func ResourceAzureBlobMount() common.Resource

ResourceAzureBlobMount creates the resource

func ResourceDatabricksMountSchema 

func ResourceDatabricksMountSchema() map[string]*schema.Schema

func ResourceDbfsFile 

func ResourceDbfsFile() common.Resource

ResourceDbfsFile manages files on DBFS

func ResourceFile added in v1.37.0 

func ResourceFile() common.Resource

func ResourceMount 

func ResourceMount() common.Resource

ResourceMount mounts using given configuration

func ValidateMountDirectory 

func ValidateMountDirectory(val any, key string) (warns []string, errs []error)

ValidateMountDirectory is a ValidateFunc that ensures the mount directory starts with a '/'

Types

type AWSIamMount 

type AWSIamMount struct {
	S3BucketName string `json:"s3_bucket_name"`
}

AWSIamMount describes the object for a aws mount using iam role

func (AWSIamMount) Config 

func (m AWSIamMount) Config(client *common.DatabricksClient) map[string]string

Config ...

func (AWSIamMount) Name 

func (m AWSIamMount) Name() string

Name ...

func (AWSIamMount) Source 

func (m AWSIamMount) Source() string

Source ...

func (AWSIamMount) ValidateAndApplyDefaults 

func (m AWSIamMount) ValidateAndApplyDefaults(d *schema.ResourceData, client *common.DatabricksClient) error

type AzureADLSGen1Mount 

type AzureADLSGen1Mount struct {
	StorageResource string `json:"storage_resource_name"`
	Directory       string `json:"directory,omitempty"`
	PrefixType      string `json:"spark_conf_prefix"`
	ClientID        string `json:"client_id"`
	TenantID        string `json:"tenant_id"`
	SecretScope     string `json:"client_secret_scope"`
	SecretKey       string `json:"client_secret_key"`
}

AzureADLSGen1Mount describes the object for a azure datalake gen 1 storage mount

func (AzureADLSGen1Mount) Config 

func (m AzureADLSGen1Mount) Config(client *common.DatabricksClient) map[string]string

Config ...

func (AzureADLSGen1Mount) Name 

func (m AzureADLSGen1Mount) Name() string

func (AzureADLSGen1Mount) Source 

func (m AzureADLSGen1Mount) Source() string

Source ...

func (AzureADLSGen1Mount) ValidateAndApplyDefaults 

func (m AzureADLSGen1Mount) ValidateAndApplyDefaults(d *schema.ResourceData, client *common.DatabricksClient) error

type AzureADLSGen1MountGeneric 

type AzureADLSGen1MountGeneric struct {
	StorageResource string `json:"storage_resource_name,omitempty" tf:"computed,force_new"`
	Directory       string `json:"directory,omitempty" tf:"force_new"`
	PrefixType      string `json:"spark_conf_prefix,omitempty" tf:"default:fs.adl,force_new"`
	ClientID        string `json:"client_id" tf:"force_new"`
	TenantID        string `json:"tenant_id,omitempty" tf:"computed,force_new"`
	SecretScope     string `json:"client_secret_scope" tf:"force_new"`
	SecretKey       string `json:"client_secret_key" tf:"force_new"`
}

AzureADLSGen1Mount describes the object for a azure datalake gen 1 storage mount

func (*AzureADLSGen1MountGeneric) Config 

func (m *AzureADLSGen1MountGeneric) Config(client *common.DatabricksClient) map[string]string

Config ...

func (*AzureADLSGen1MountGeneric) Name 

func (m *AzureADLSGen1MountGeneric) Name() string

func (*AzureADLSGen1MountGeneric) Source 

func (m *AzureADLSGen1MountGeneric) Source() string

Source ...

func (*AzureADLSGen1MountGeneric) ValidateAndApplyDefaults 

func (m *AzureADLSGen1MountGeneric) ValidateAndApplyDefaults(d *schema.ResourceData, client *common.DatabricksClient) error

type AzureADLSGen2Mount 

type AzureADLSGen2Mount struct {
	ContainerName        string `json:"container_name"`
	StorageAccountName   string `json:"storage_account_name"`
	Directory            string `json:"directory,omitempty"`
	ClientID             string `json:"client_id"`
	TenantID             string `json:"tenant_id"`
	SecretScope          string `json:"client_secret_scope"`
	SecretKey            string `json:"client_secret_key"`
	InitializeFileSystem bool   `json:"initialize_file_system"`
}

AzureADLSGen2Mount describes the object for a azure datalake gen 2 storage mount

func (AzureADLSGen2Mount) Config 

func (m AzureADLSGen2Mount) Config(client *common.DatabricksClient) map[string]string

Config returns mount configurations

func (AzureADLSGen2Mount) Name 

func (m AzureADLSGen2Mount) Name() string

func (AzureADLSGen2Mount) Source 

func (m AzureADLSGen2Mount) Source() string

Source returns ABFSS URI backing the mount

func (AzureADLSGen2Mount) ValidateAndApplyDefaults 

func (m AzureADLSGen2Mount) ValidateAndApplyDefaults(d *schema.ResourceData, client *common.DatabricksClient) error

type AzureADLSGen2MountGeneric 

type AzureADLSGen2MountGeneric struct {
	ContainerName        string `json:"container_name,omitempty" tf:"computed,force_new"`
	StorageAccountName   string `json:"storage_account_name,omitempty" tf:"computed,force_new"`
	Directory            string `json:"directory,omitempty" tf:"force_new"`
	ClientID             string `json:"client_id" tf:"force_new"`
	TenantID             string `json:"tenant_id,omitempty" tf:"computed,force_new"`
	SecretScope          string `json:"client_secret_scope" tf:"force_new"`
	SecretKey            string `json:"client_secret_key" tf:"force_new"`
	InitializeFileSystem bool   `json:"initialize_file_system" tf:"force_new"`
}

AzureADLSGen2Mount describes the object for a azure datalake gen 2 storage mount

func (*AzureADLSGen2MountGeneric) Config 

func (m *AzureADLSGen2MountGeneric) Config(client *common.DatabricksClient) map[string]string

Config returns mount configurations

func (*AzureADLSGen2MountGeneric) Name 

func (m *AzureADLSGen2MountGeneric) Name() string

func (*AzureADLSGen2MountGeneric) Source 

func (m *AzureADLSGen2MountGeneric) Source() string

Source returns ABFSS URI backing the mount

func (*AzureADLSGen2MountGeneric) ValidateAndApplyDefaults 

func (m *AzureADLSGen2MountGeneric) ValidateAndApplyDefaults(d *schema.ResourceData, client *common.DatabricksClient) error

type AzureBlobMount 

type AzureBlobMount struct {
	ContainerName      string `json:"container_name"`
	StorageAccountName string `json:"storage_account_name"`
	Directory          string `json:"directory"`
	AuthType           string `json:"auth_type"`
	SecretScope        string `json:"token_secret_scope"`
	SecretKey          string `json:"token_secret_key"`
}

AzureBlobMount describes the object for a azure blob storage mount - a.k.a. NativeAzureFileSystem

func (AzureBlobMount) Config 

func (m AzureBlobMount) Config(client *common.DatabricksClient) map[string]string

Config ...

func (AzureBlobMount) Name 

func (m AzureBlobMount) Name() string

func (AzureBlobMount) Source 

func (m AzureBlobMount) Source() string

Source ...

func (AzureBlobMount) ValidateAndApplyDefaults 

func (m AzureBlobMount) ValidateAndApplyDefaults(d *schema.ResourceData, client *common.DatabricksClient) error

type AzureBlobMountGeneric 

type AzureBlobMountGeneric struct {
	ContainerName      string `json:"container_name,omitempty" tf:"computed,force_new"`
	StorageAccountName string `json:"storage_account_name,omitempty" tf:"computed,force_new"`
	Directory          string `json:"directory,omitempty" tf:"force_new"`
	AuthType           string `json:"auth_type" tf:"force_new"`
	SecretScope        string `json:"token_secret_scope" tf:"force_new"`
	SecretKey          string `json:"token_secret_key" tf:"force_new"`
}

AzureBlobMount describes the object for a azure blob storage mount - a.k.a. NativeAzureFileSystem

func (*AzureBlobMountGeneric) Config 

func (m *AzureBlobMountGeneric) Config(client *common.DatabricksClient) map[string]string

Config ...

func (*AzureBlobMountGeneric) Name 

func (m *AzureBlobMountGeneric) Name() string

func (*AzureBlobMountGeneric) Source 

func (m *AzureBlobMountGeneric) Source() string

Source ...

func (*AzureBlobMountGeneric) ValidateAndApplyDefaults 

func (m *AzureBlobMountGeneric) ValidateAndApplyDefaults(d *schema.ResourceData, client *common.DatabricksClient) error

type DbfsAPI 

type DbfsAPI struct {
	// contains filtered or unexported fields
}

DbfsAPI exposes the DBFS API

func NewDbfsAPI 

func NewDbfsAPI(ctx context.Context, m any) DbfsAPI

NewDbfsAPI creates DBFSAPI instance from provider meta

func (DbfsAPI) Create 

func (a DbfsAPI) Create(path string, contents []byte, overwrite bool) (err error)

Create creates a file on DBFS

func (DbfsAPI) Delete 

func (a DbfsAPI) Delete(path string, recursive bool) error

Delete deletes a file in DBFS via API

func (DbfsAPI) List 

func (a DbfsAPI) List(path string, recursive bool) ([]FileInfo, error)

List returns a list of files in DBFS and the recursive flag lets you recursively list files

func (DbfsAPI) Read 

func (a DbfsAPI) Read(path string) (content []byte, err error)

Read returns the contents of a file

func (DbfsAPI) Status 

func (a DbfsAPI) Status(path string) (f FileInfo, err error)

Status returns the status of a file in DBFS

type FileInfo 

type FileInfo struct {
	Path     string `json:"path,omitempty"`
	IsDir    bool   `json:"is_dir,omitempty"`
	FileSize int64  `json:"file_size,omitempty"`
}

FileInfo contains information when listing files or fetching files from DBFS api

type FileList 

type FileList struct {
	Files []FileInfo `json:"files,omitempty"`
}

FileList contains list of file metadata entries

type GSMount 

type GSMount struct {
	BucketName     string `json:"bucket_name" tf:"force_new"`
	ServiceAccount string `json:"service_account,omitempty" tf:"force_new"`
}

GSMount describes the object for a GS mount using google service account

func (GSMount) Config 

func (m GSMount) Config(client *common.DatabricksClient) map[string]string

Config ...

func (GSMount) Name 

func (m GSMount) Name() string

func (GSMount) Source 

func (m GSMount) Source() string

Source ...

func (GSMount) ValidateAndApplyDefaults 

func (m GSMount) ValidateAndApplyDefaults(d *schema.ResourceData, client *common.DatabricksClient) error

type GenericMount 

type GenericMount struct {
	URI     string                     `json:"uri,omitempty" tf:"force_new"`
	Options map[string]string          `json:"extra_configs,omitempty" tf:"force_new"`
	Abfs    *AzureADLSGen2MountGeneric `json:"abfs,omitempty" tf:"force_new,suppress_diff"`
	S3      *S3IamMount                `json:"s3,omitempty" tf:"force_new,suppress_diff"`
	Adl     *AzureADLSGen1MountGeneric `json:"adl,omitempty" tf:"force_new,suppress_diff"`
	Wasb    *AzureBlobMountGeneric     `json:"wasb,omitempty" tf:"force_new,suppress_diff"`
	Gs      *GSMount                   `json:"gs,omitempty" tf:"force_new,suppress_diff"`

	ClusterID      string `json:"cluster_id,omitempty" tf:"computed,force_new"`
	MountName      string `json:"name,omitempty" tf:"computed,force_new"`
	ResourceID     string `json:"resource_id,omitempty" tf:"force_new"`
	EncryptionType string `json:"encryption_type,omitempty" tf:"force_new"`
}

TODO: add support for encryption parameters in S3

func (GenericMount) Config 

func (m GenericMount) Config(client *common.DatabricksClient) map[string]string

Config returns mount configurations

func (GenericMount) Name 

func (m GenericMount) Name() string

Name

func (GenericMount) Source 

func (m GenericMount) Source() string

Source returns URI backing the mount

func (GenericMount) ValidateAndApplyDefaults 

func (m GenericMount) ValidateAndApplyDefaults(d *schema.ResourceData, client *common.DatabricksClient) error

ApplyDefaults tries to apply defaults to a given resource

type Mount 

type Mount interface {
	Source() string
	Config(client *common.DatabricksClient) map[string]string

	Name() string
	ValidateAndApplyDefaults(d *schema.ResourceData, client *common.DatabricksClient) error
}

Mount exposes generic url & extra config map options

type MountPoint 

type MountPoint struct {
	Exec           common.CommandExecutor
	ClusterID      string
	Name           string
	EncryptionType string
}

MountPoint is something actionable

func NewMountPoint 

func NewMountPoint(executor common.CommandExecutor, name, clusterID string) MountPoint

NewMountPoint returns new mount point config

func (MountPoint) Delete 

func (mp MountPoint) Delete() error

Delete removes mount from workspace

func (MountPoint) Mount 

func (mp MountPoint) Mount(mo Mount, client *common.DatabricksClient) (source string, err error)

Mount mounts object store on workspace

func (MountPoint) Source 

func (mp MountPoint) Source() (string, error)

Source returns mountpoint source

type ReadResponse 

type ReadResponse struct {
	BytesRead int64  `json:"bytes_read"`
	Data      string `json:"data"`
}

ReadResponse contains the response from reading a portion of a file in DBFS

type S3IamMount 

type S3IamMount struct {
	BucketName      string `json:"bucket_name" tf:"force_new"`
	InstanceProfile string `json:"instance_profile,omitempty" tf:"force_new"`
}

S3IamMount describes the object for a aws mount using iam role

func (S3IamMount) Config 

func (m S3IamMount) Config(client *common.DatabricksClient) map[string]string

Config ...

func (S3IamMount) Name 

func (m S3IamMount) Name() string

Name ...

func (S3IamMount) Source 

func (m S3IamMount) Source() string

Source ...

func (S3IamMount) ValidateAndApplyDefaults 

func (m S3IamMount) ValidateAndApplyDefaults(d *schema.ResourceData, client *common.DatabricksClient) error

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.