azstorage

package
v1.7.1 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Nov 8, 2024 License: MIT Imports: 50 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	EnvAzStorageAccount                = "AZURE_STORAGE_ACCOUNT"
	EnvAzStorageAccountType            = "AZURE_STORAGE_ACCOUNT_TYPE"
	EnvAzStorageAccessKey              = "AZURE_STORAGE_ACCESS_KEY"
	EnvAzStorageSasToken               = "AZURE_STORAGE_SAS_TOKEN"
	EnvAzStorageIdentityClientId       = "AZURE_STORAGE_IDENTITY_CLIENT_ID"
	EnvAzStorageIdentityResourceId     = "AZURE_STORAGE_IDENTITY_RESOURCE_ID"
	EnvAzStorageIdentityObjectId       = "AZURE_STORAGE_IDENTITY_OBJECT_ID"
	EnvAzStorageSpnTenantId            = "AZURE_STORAGE_SPN_TENANT_ID"
	EnvAzStorageSpnClientId            = "AZURE_STORAGE_SPN_CLIENT_ID"
	EnvAzStorageSpnClientSecret        = "AZURE_STORAGE_SPN_CLIENT_SECRET"
	EnvAzStorageSpnOAuthTokenFilePath  = "AZURE_OAUTH_TOKEN_FILE"
	EnvAzStorageAadEndpoint            = "AZURE_STORAGE_AAD_ENDPOINT"
	EnvAzStorageAuthType               = "AZURE_STORAGE_AUTH_TYPE"
	EnvAzStorageBlobEndpoint           = "AZURE_STORAGE_BLOB_ENDPOINT"
	EnvAzStorageAccountContainer       = "AZURE_STORAGE_ACCOUNT_CONTAINER"
	EnvAzAuthResource                  = "AZURE_STORAGE_AUTH_RESOURCE"
	EnvAzStorageCpkEncryptionKey       = "AZURE_STORAGE_CPK_ENCRYPTION_KEY"
	EnvAzStorageCpkEncryptionKeySha256 = "AZURE_STORAGE_CPK_ENCRYPTION_KEY_SHA256"
)

Environment variable names Here we are not reading MSI_ENDPOINT and MSI_SECRET as they are read by go-sdk directly https://github.com/Azure/go-autorest/blob/a46566dfcbdc41e736295f94e9f690ceaf50094a/autorest/adal/token.go#L788 newServicePrincipalTokenFromMSI : reads them directly from env

View Source 
const (
	Timeout                time.Duration = 30 * time.Second
	KeepAlive              time.Duration = 30 * time.Second
	DualStack              bool          = true
	MaxIdleConns           int           = 0 // No limit
	MaxIdleConnsPerHost    int           = 100
	MaxConnsPerHost        int           = 0 // No limit
	IdleConnTimeout        time.Duration = 90 * time.Second
	TLSHandshakeTimeout    time.Duration = 10 * time.Second
	ExpectContinueTimeout  time.Duration = 1 * time.Second
	DisableKeepAlives      bool          = false
	DisableCompression     bool          = false
	MaxResponseHeaderBytes int64         = 0
)
View Source 
const (
	ErrNoErr uint16 = iota
	ErrUnknown
	ErrFileNotFound
	ErrFileAlreadyExists
	InvalidRange
	BlobIsUnderLease
	InvalidPermission
)

----------- Store error code handling ---------------

View Source 
const DefaultMaxResultsForList int32 = 2

default value for maximum results returned by a list API call

View Source 
const (
	MaxBlobSize = blockblob.MaxStageBlockBytes * blockblob.MaxBlocks
)

Variables

View Source 
var AccessTiers = map[string]blob.AccessTier{
	"hot":     blob.AccessTierHot,
	"cool":    blob.AccessTierCool,
	"cold":    blob.AccessTierCold,
	"archive": blob.AccessTierArchive,
	"p4":      blob.AccessTierP4,
	"p6":      blob.AccessTierP6,
	"p10":     blob.AccessTierP10,
	"p15":     blob.AccessTierP15,
	"p20":     blob.AccessTierP20,
	"p30":     blob.AccessTierP30,
	"p40":     blob.AccessTierP40,
	"p50":     blob.AccessTierP50,
	"p60":     blob.AccessTierP60,
	"p70":     blob.AccessTierP70,
	"p80":     blob.AccessTierP80,
	"premium": blob.AccessTierPremium,
}
----------- Blob access tier type conversion  ---------------

AccessTierMap : Store config to access tier mapping

View Source 
var ContentTypes = map[string]string{
	".css":  "text/css",
	".pdf":  "application/pdf",
	".xml":  "text/xml",
	".csv":  "text/csv",
	".json": "application/json",
	".rtf":  "application/rtf",
	".txt":  "text/plain",
	".java": "text/plain",
	".dat":  "text/plain",

	".htm":  "text/html",
	".html": "text/html",

	".gif":  "image/gif",
	".jpeg": "image/jpeg",
	".jpg":  "image/jpeg",
	".png":  "image/png",
	".bmp":  "image/bmp",

	".js":   "application/javascript",
	".mjs":  "application/javascript",
	".svg":  "image/svg+xml",
	".wasm": "application/wasm",
	".webp": "image/webp",

	".wav":  "audio/wav",
	".mp3":  "audio/mpeg",
	".mpeg": "video/mpeg",
	".aac":  "audio/aac",
	".avi":  "video/x-msvideo",
	".m3u8": "application/x-mpegURL",
	".ts":   "video/MP2T",
	".mid":  "audio/midiaudio/x-midi",
	".3gp":  "video/3gpp",
	".mp4":  "video/mp4",

	".doc":  "application/msword",
	".docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
	".ppt":  "application/vnd.ms-powerpoint",
	".pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation",
	".xls":  "application/vnd.ms-excel",
	".xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",

	".gz":   "application/x-gzip",
	".jar":  "application/java-archive",
	".rar":  "application/vnd.rar",
	".tar":  "application/x-tar",
	".zip":  "application/x-zip-compressed",
	".7z":   "application/x-7z-compressed",
	".3g2":  "video/3gpp2",
	".usdz": "application/zip",

	".sh":  "application/x-sh",
	".exe": "application/x-msdownload",
	".dll": "application/x-msdownload",
}

ContentTypeMap : Store file extension to content-type mapping

View Source 
var EAccountType = AccountType(0).INVALID_ACC()
View Source 
var EAuthType = AuthType(0).INVALID_AUTH()
View Source 
var UserAgent = func() string {
	return "Seagate-Cloudfuse/" + common.CloudfuseVersion
}

Functions

func NewazstorageComponent 

func NewazstorageComponent() internal.Component

Constructor to create object of this component

func ParseAndReadDynamicConfig 

func ParseAndReadDynamicConfig(az *AzStorage, opt AzStorageOptions, reload bool) error

ParseAndReadDynamicConfig : On config change read only the required config

func ParseAndValidateConfig 

func ParseAndValidateConfig(az *AzStorage, opt AzStorageOptions) error

ParseAndValidateConfig : Parse and validate config

func RegisterEnvVariables 

func RegisterEnvVariables()

RegisterEnvVariables : Register environment variables

Types

type AccountType 

type AccountType int

AccountType Enum

func (AccountType) ADLS 

func (AccountType) ADLS() AccountType

func (AccountType) BLOCK 

func (AccountType) BLOCK() AccountType

func (AccountType) INVALID_ACC 

func (AccountType) INVALID_ACC() AccountType

func (*AccountType) Parse 

func (a *AccountType) Parse(s string) error

func (AccountType) String 

func (f AccountType) String() string

type AuthType 

type AuthType int

AuthType Enum

func (AuthType) AZCLI added in v1.3.0 

func (AuthType) AZCLI() AuthType

func (AuthType) INVALID_AUTH 

func (AuthType) INVALID_AUTH() AuthType

func (AuthType) KEY 

func (AuthType) KEY() AuthType

func (AuthType) MSI 

func (AuthType) MSI() AuthType

func (*AuthType) Parse 

func (a *AuthType) Parse(s string) error

func (AuthType) SAS 

func (AuthType) SAS() AuthType

func (AuthType) SPN 

func (AuthType) SPN() AuthType

func (AuthType) String 

func (a AuthType) String() string

type AzConnection 

type AzConnection interface {
	Configure(cfg AzStorageConfig) error
	UpdateConfig(cfg AzStorageConfig) error

	SetupPipeline() error
	TestPipeline() error

	ListContainers() ([]string, error)

	// This is just for test, shall not be used otherwise
	SetPrefixPath(string) error

	CreateFile(name string, mode os.FileMode) error
	CreateDirectory(name string) error
	CreateLink(source string, target string) error

	DeleteFile(name string) error
	DeleteDirectory(name string) error

	RenameFile(string, string) error
	RenameDirectory(string, string) error

	GetAttr(name string) (attr *internal.ObjAttr, err error)

	// Standard operations to be supported by any account type
	List(prefix string, marker *string, count int32) ([]*internal.ObjAttr, *string, error)

	ReadToFile(name string, offset int64, count int64, fi *os.File) error
	ReadBuffer(name string, offset int64, length int64) ([]byte, error)
	ReadInBuffer(name string, offset int64, length int64, data []byte) error

	WriteFromFile(name string, metadata map[string]*string, fi *os.File) error
	WriteFromBuffer(name string, metadata map[string]*string, data []byte) error
	Write(options internal.WriteFileOptions) error
	GetFileBlockOffsets(name string) (*common.BlockOffsetList, error)

	ChangeMod(string, os.FileMode) error
	ChangeOwner(string, int, int) error
	TruncateFile(string, int64) error
	StageAndCommit(name string, bol *common.BlockOffsetList) error

	GetCommittedBlockList(string) (*internal.CommittedBlockList, error)
	StageBlock(string, []byte, string) error
	CommitBlocks(string, []string) error

	UpdateServiceClient(_, _ string) error
}

func NewAzStorageConnection 

func NewAzStorageConnection(cfg AzStorageConfig) AzConnection

NewAzStorageConnection : Based on account type create respective AzConnection Object

type AzStorage 

type AzStorage struct {
	internal.BaseComponent
	// contains filtered or unexported fields
}

AzStorage Wrapper type around azure go-sdk (track-1)

func (*AzStorage) Chmod 

func (az *AzStorage) Chmod(options internal.ChmodOptions) error

func (*AzStorage) Chown 

func (az *AzStorage) Chown(options internal.ChownOptions) error

func (*AzStorage) CloseFile 

func (az *AzStorage) CloseFile(options internal.CloseFileOptions) error

func (*AzStorage) CommitData added in v1.1.0 

func (az *AzStorage) CommitData(opt internal.CommitDataOptions) error

func (*AzStorage) Configure 

func (az *AzStorage) Configure(isParent bool) error

Configure : Pipeline will call this method after constructor so that you can read config and initialize yourself

func (*AzStorage) CopyFromFile 

func (az *AzStorage) CopyFromFile(options internal.CopyFromFileOptions) error

func (*AzStorage) CopyToFile 

func (az *AzStorage) CopyToFile(options internal.CopyToFileOptions) error

func (*AzStorage) CreateDir 

func (az *AzStorage) CreateDir(options internal.CreateDirOptions) error

Directory operations

func (*AzStorage) CreateFile 

func (az *AzStorage) CreateFile(options internal.CreateFileOptions) (*handlemap.Handle, error)

File operations 

func (az *AzStorage) CreateLink(options internal.CreateLinkOptions) error

Symlink operations

func (*AzStorage) DeleteDir 

func (az *AzStorage) DeleteDir(options internal.DeleteDirOptions) error

func (*AzStorage) DeleteFile 

func (az *AzStorage) DeleteFile(options internal.DeleteFileOptions) error

func (*AzStorage) FlushFile 

func (az *AzStorage) FlushFile(options internal.FlushFileOptions) error

func (*AzStorage) GetAttr 

func (az *AzStorage) GetAttr(options internal.GetAttrOptions) (attr *internal.ObjAttr, err error)

Attribute operations

func (*AzStorage) GetCommittedBlockList added in v1.1.0 

func (az *AzStorage) GetCommittedBlockList(name string) (*internal.CommittedBlockList, error)

func (*AzStorage) GetFileBlockOffsets 

func (az *AzStorage) GetFileBlockOffsets(options internal.GetFileBlockOffsetsOptions) (*common.BlockOffsetList, error)

func (*AzStorage) IsDirEmpty 

func (az *AzStorage) IsDirEmpty(options internal.IsDirEmptyOptions) bool

func (*AzStorage) ListContainers 

func (az *AzStorage) ListContainers() ([]string, error)

------------------------- Container listing -------------------------------------------

func (*AzStorage) Name 

func (az *AzStorage) Name() string

func (*AzStorage) OnConfigChange 

func (az *AzStorage) OnConfigChange()

OnConfigChange : When config file is changed, this will be called by pipeline. Refresh required config here

func (*AzStorage) OpenFile 

func (az *AzStorage) OpenFile(options internal.OpenFileOptions) (*handlemap.Handle, error)

func (*AzStorage) Priority 

func (az *AzStorage) Priority() internal.ComponentPriority

func (*AzStorage) ReadInBuffer 

func (az *AzStorage) ReadInBuffer(options internal.ReadInBufferOptions) (length int, err error)
func (az *AzStorage) ReadLink(options internal.ReadLinkOptions) (string, error)

func (*AzStorage) RenameDir 

func (az *AzStorage) RenameDir(options internal.RenameDirOptions) error

func (*AzStorage) RenameFile 

func (az *AzStorage) RenameFile(options internal.RenameFileOptions) error

func (*AzStorage) SetName 

func (az *AzStorage) SetName(name string)

func (*AzStorage) SetNextComponent 

func (az *AzStorage) SetNextComponent(c internal.Component)

func (*AzStorage) StageData added in v1.1.0 

func (az *AzStorage) StageData(opt internal.StageDataOptions) error

func (*AzStorage) Start 

func (az *AzStorage) Start(ctx context.Context) error

Start : Initialize the go-sdk pipeline here and test auth is working fine

func (*AzStorage) Stop 

func (az *AzStorage) Stop() error

Stop : Disconnect all running operations here

func (*AzStorage) StreamDir 

func (az *AzStorage) StreamDir(options internal.StreamDirOptions) ([]*internal.ObjAttr, string, error)

func (*AzStorage) TruncateFile 

func (az *AzStorage) TruncateFile(options internal.TruncateFileOptions) error

func (*AzStorage) WriteFile 

func (az *AzStorage) WriteFile(options internal.WriteFileOptions) (int, error)

type AzStorageConfig 

type AzStorageConfig struct {
	// contains filtered or unexported fields
}

Example for azblob usage : https://pkg.go.dev/github.com/Azure/azure-sdk-for-go/sdk/storage/azblob#pkg-examples For methods help refer : https://pkg.go.dev/github.com/Azure/azure-sdk-for-go/sdk/storage/azblob#Client

type AzStorageConnection 

type AzStorageConnection struct {
	Config AzStorageConfig
}

type AzStorageOptions 

type AzStorageOptions struct {
	AccountType             string `config:"type" yaml:"type,omitempty"`
	UseHTTP                 bool   `config:"use-http" yaml:"use-http,omitempty"`
	AccountName             string `config:"account-name" yaml:"account-name,omitempty"`
	AccountKey              string `config:"account-key" yaml:"account-key,omitempty"`
	SaSKey                  string `config:"sas" yaml:"sas,omitempty"`
	ApplicationID           string `config:"appid" yaml:"appid,omitempty"`
	ResourceID              string `config:"resid" yaml:"resid,omitempty"`
	ObjectID                string `config:"objid" yaml:"objid,omitempty"`
	TenantID                string `config:"tenantid" yaml:"tenantid,omitempty"`
	ClientID                string `config:"clientid" yaml:"clientid,omitempty"`
	ClientSecret            string `config:"clientsecret" yaml:"clientsecret,omitempty"`
	OAuthTokenFilePath      string `config:"oauth-token-path" yaml:"oauth-token-path,omitempty"`
	ActiveDirectoryEndpoint string `config:"aadendpoint" yaml:"aadendpoint,omitempty"`
	Endpoint                string `config:"endpoint" yaml:"endpoint,omitempty"`
	AuthMode                string `config:"mode" yaml:"mode,omitempty"`
	Container               string `config:"container" yaml:"container,omitempty"`
	PrefixPath              string `config:"subdirectory" yaml:"subdirectory,omitempty"`
	BlockSize               int64  `config:"block-size-mb" yaml:"block-size-mb,omitempty"`
	MaxConcurrency          uint16 `config:"max-concurrency" yaml:"max-concurrency,omitempty"`
	DefaultTier             string `config:"tier" yaml:"tier,omitempty"`
	CancelListForSeconds    uint16 `config:"block-list-on-mount-sec" yaml:"block-list-on-mount-sec,omitempty"`
	MaxRetries              int32  `config:"max-retries" yaml:"max-retries,omitempty"`
	MaxTimeout              int32  `config:"max-retry-timeout-sec" yaml:"max-retry-timeout-sec,omitempty"`
	BackoffTime             int32  `config:"retry-backoff-sec" yaml:"retry-backoff-sec,omitempty"`
	MaxRetryDelay           int32  `config:"max-retry-delay-sec" yaml:"max-retry-delay-sec,omitempty"`
	HttpProxyAddress        string `config:"http-proxy" yaml:"http-proxy,omitempty"`
	HttpsProxyAddress       string `config:"https-proxy" yaml:"https-proxy,omitempty"`
	FailUnsupportedOp       bool   `config:"fail-unsupported-op" yaml:"fail-unsupported-op,omitempty"`
	AuthResourceString      string `config:"auth-resource" yaml:"auth-resource,omitempty"`
	UpdateMD5               bool   `config:"update-md5" yaml:"update-md5"`
	ValidateMD5             bool   `config:"validate-md5" yaml:"validate-md5"`
	VirtualDirectory        bool   `config:"virtual-directory" yaml:"virtual-directory"`
	MaxResultsForList       int32  `config:"max-results-for-list" yaml:"max-results-for-list"`
	DisableCompression      bool   `config:"disable-compression" yaml:"disable-compression"`
	Telemetry               string `config:"telemetry" yaml:"telemetry"`
	HonourACL               bool   `config:"honour-acl" yaml:"honour-acl"`
	RestrictedCharsWin      bool   `config:"restricted-characters-windows" yaml:"-"`
	CPKEnabled              bool   `config:"cpk-enabled" yaml:"cpk-enabled"`
	CPKEncryptionKey        string `config:"cpk-encryption-key" yaml:"cpk-encryption-key"`
	CPKEncryptionKeySha256  string `config:"cpk-encryption-key-sha256" yaml:"cpk-encryption-key-sha256"`

	// v1 support
	UseAdls        bool   `config:"use-adls" yaml:"-"`
	UseHTTPS       bool   `config:"use-https" yaml:"-"`
	SetContentType bool   `config:"set-content-type" yaml:"-"`
	CaCertFile     string `config:"ca-cert-file" yaml:"-"`
}

type BlockBlob 

type BlockBlob struct {
	AzStorageConnection
	Auth      azAuth
	Service   *service.Client
	Container *container.Client
	// contains filtered or unexported fields
}

func (*BlockBlob) ChangeMod 

func (bb *BlockBlob) ChangeMod(name string, _ os.FileMode) error

ChangeMod : Change mode of a blob

func (*BlockBlob) ChangeOwner 

func (bb *BlockBlob) ChangeOwner(name string, _ int, _ int) error

ChangeOwner : Change owner of a blob

func (*BlockBlob) CommitBlocks added in v1.1.0 

func (bb *BlockBlob) CommitBlocks(name string, blockList []string) error

CommitBlocks : persists the block list

func (*BlockBlob) Configure 

func (bb *BlockBlob) Configure(cfg AzStorageConfig) error

func (*BlockBlob) CreateDirectory 

func (bb *BlockBlob) CreateDirectory(name string) error

CreateDirectory : Create a new directory in the container/virtual directory

func (*BlockBlob) CreateFile 

func (bb *BlockBlob) CreateFile(name string, mode os.FileMode) error

CreateFile : Create a new file in the container/virtual directory 

func (bb *BlockBlob) CreateLink(source string, target string) error

CreateLink : Create a symlink in the container/virtual directory

func (*BlockBlob) DeleteDirectory 

func (bb *BlockBlob) DeleteDirectory(name string) (err error)

DeleteDirectory : Delete a virtual directory in the container/virtual directory

func (*BlockBlob) DeleteFile 

func (bb *BlockBlob) DeleteFile(name string) (err error)

DeleteFile : Delete a blob in the container/virtual directory

func (*BlockBlob) GetAttr 

func (bb *BlockBlob) GetAttr(name string) (attr *internal.ObjAttr, err error)

GetAttr : Retrieve attributes of the blob

func (*BlockBlob) GetCommittedBlockList added in v1.1.0 

func (bb *BlockBlob) GetCommittedBlockList(name string) (*internal.CommittedBlockList, error)

GetCommittedBlockList : Get the list of committed blocks

func (*BlockBlob) GetFileBlockOffsets 

func (bb *BlockBlob) GetFileBlockOffsets(name string) (*common.BlockOffsetList, error)

GetFileBlockOffsets: store blocks ids and corresponding offsets

func (*BlockBlob) HandleSmallFile added in v0.3.0 

func (bb *BlockBlob) HandleSmallFile(name string, size int64, originalSize int64) ([]byte, error)

func (*BlockBlob) List 

func (bb *BlockBlob) List(prefix string, marker *string, count int32) ([]*internal.ObjAttr, *string, error)

List : Get a list of blobs matching the given prefix This fetches the list using a marker so the caller code should handle marker logic If count=0 - fetch max entries

func (*BlockBlob) ListContainers 

func (bb *BlockBlob) ListContainers() ([]string, error)

func (*BlockBlob) ReadBuffer 

func (bb *BlockBlob) ReadBuffer(name string, offset int64, length int64) ([]byte, error)

ReadBuffer : Download a specific range from a blob to a buffer

func (*BlockBlob) ReadInBuffer 

func (bb *BlockBlob) ReadInBuffer(name string, offset int64, length int64, data []byte) error

ReadInBuffer : Download specific range from a file to a user provided buffer

func (*BlockBlob) ReadToFile 

func (bb *BlockBlob) ReadToFile(name string, offset int64, count int64, fi *os.File) (err error)

ReadToFile : Download a blob to a local file

func (*BlockBlob) RenameDirectory 

func (bb *BlockBlob) RenameDirectory(source string, target string) error

RenameDirectory : Rename the directory

func (*BlockBlob) RenameFile 

func (bb *BlockBlob) RenameFile(source string, target string) error

RenameFile : Rename the file

func (*BlockBlob) SetPrefixPath 

func (bb *BlockBlob) SetPrefixPath(path string) error

func (*BlockBlob) SetupPipeline 

func (bb *BlockBlob) SetupPipeline() error

SetupPipeline : Based on the config setup the ***URLs

func (*BlockBlob) StageAndCommit 

func (bb *BlockBlob) StageAndCommit(name string, bol *common.BlockOffsetList) error

func (*BlockBlob) StageBlock added in v1.1.0 

func (bb *BlockBlob) StageBlock(name string, data []byte, id string) error

StageBlock : stages a block and returns its blockid

func (*BlockBlob) TestPipeline 

func (bb *BlockBlob) TestPipeline() error

TestPipeline : Validate the credentials specified in the auth config

func (*BlockBlob) TruncateFile 

func (bb *BlockBlob) TruncateFile(name string, size int64) error

func (*BlockBlob) UpdateConfig 

func (bb *BlockBlob) UpdateConfig(cfg AzStorageConfig) error

For dynamic config update the config here

func (*BlockBlob) UpdateServiceClient added in v1.3.0 

func (bb *BlockBlob) UpdateServiceClient(key, value string) (err error)

UpdateServiceClient : Update the SAS specified by the user and create new service client

func (*BlockBlob) Write 

func (bb *BlockBlob) Write(options internal.WriteFileOptions) error

Write : write data at given offset to a blob

func (*BlockBlob) WriteFromBuffer 

func (bb *BlockBlob) WriteFromBuffer(name string, metadata map[string]*string, data []byte) error

WriteFromBuffer : Upload from a buffer to a blob

func (*BlockBlob) WriteFromFile 

func (bb *BlockBlob) WriteFromFile(name string, metadata map[string]*string, fi *os.File) (err error)

WriteFromFile : Upload local file to blob

type Datalake 

type Datalake struct {
	AzStorageConnection
	Auth       azAuth
	Service    *service.Client
	Filesystem *filesystem.Client
	BlockBlob  BlockBlob
	// contains filtered or unexported fields
}

func (*Datalake) ChangeMod 

func (dl *Datalake) ChangeMod(name string, mode os.FileMode) error

ChangeMod : Change mode of a path

func (*Datalake) ChangeOwner 

func (dl *Datalake) ChangeOwner(name string, _ int, _ int) error

ChangeOwner : Change owner of a path

func (*Datalake) CommitBlocks added in v1.1.0 

func (dl *Datalake) CommitBlocks(name string, blockList []string) error

CommitBlocks : persists the block list

func (*Datalake) Configure 

func (dl *Datalake) Configure(cfg AzStorageConfig) error

func (*Datalake) CreateDirectory 

func (dl *Datalake) CreateDirectory(name string) error

CreateDirectory : Create a new directory in the filesystem/directory

func (*Datalake) CreateFile 

func (dl *Datalake) CreateFile(name string, mode os.FileMode) error

CreateFile : Create a new file in the filesystem/directory 

func (dl *Datalake) CreateLink(source string, target string) error

CreateLink : Create a symlink in the filesystem/directory

func (*Datalake) DeleteDirectory 

func (dl *Datalake) DeleteDirectory(name string) (err error)

DeleteDirectory : Delete a directory in the filesystem/directory

func (*Datalake) DeleteFile 

func (dl *Datalake) DeleteFile(name string) (err error)

DeleteFile : Delete a file in the filesystem/directory

func (*Datalake) GetAttr 

func (dl *Datalake) GetAttr(name string) (attr *internal.ObjAttr, err error)

GetAttr : Retrieve attributes of the path

func (*Datalake) GetCommittedBlockList added in v1.1.0 

func (dl *Datalake) GetCommittedBlockList(name string) (*internal.CommittedBlockList, error)

GetCommittedBlockList : Get the list of committed blocks

func (*Datalake) GetFileBlockOffsets 

func (dl *Datalake) GetFileBlockOffsets(name string) (*common.BlockOffsetList, error)

func (*Datalake) List 

func (dl *Datalake) List(prefix string, marker *string, count int32) ([]*internal.ObjAttr, *string, error)

List : Get a list of path matching the given prefix This fetches the list using a marker so the caller code should handle marker logic If count=0 - fetch max entries

func (*Datalake) ListContainers 

func (dl *Datalake) ListContainers() ([]string, error)

func (*Datalake) ReadBuffer 

func (dl *Datalake) ReadBuffer(name string, offset int64, length int64) ([]byte, error)

ReadBuffer : Download a specific range from a file to a buffer

func (*Datalake) ReadInBuffer 

func (dl *Datalake) ReadInBuffer(name string, offset int64, length int64, data []byte) error

ReadInBuffer : Download specific range from a file to a user provided buffer

func (*Datalake) ReadToFile 

func (dl *Datalake) ReadToFile(name string, offset int64, count int64, fi *os.File) (err error)

ReadToFile : Download a file to a local file

func (*Datalake) RenameDirectory 

func (dl *Datalake) RenameDirectory(source string, target string) error

RenameDirectory : Rename the directory

func (*Datalake) RenameFile 

func (dl *Datalake) RenameFile(source string, target string) error

RenameFile : Rename the file

func (*Datalake) SetPrefixPath 

func (dl *Datalake) SetPrefixPath(path string) error

func (*Datalake) SetupPipeline 

func (dl *Datalake) SetupPipeline() error

SetupPipeline : Based on the config setup the ***URLs

func (*Datalake) StageAndCommit 

func (dl *Datalake) StageAndCommit(name string, bol *common.BlockOffsetList) error

func (*Datalake) StageBlock added in v1.1.0 

func (dl *Datalake) StageBlock(name string, data []byte, id string) error

StageBlock : stages a block and returns its blockid

func (*Datalake) TestPipeline 

func (dl *Datalake) TestPipeline() error

TestPipeline : Validate the credentials specified in the auth config

func (*Datalake) TruncateFile 

func (dl *Datalake) TruncateFile(name string, size int64) error

func (*Datalake) UpdateConfig 

func (dl *Datalake) UpdateConfig(cfg AzStorageConfig) error

For dynamic config update the config here

func (*Datalake) UpdateServiceClient added in v1.3.0 

func (dl *Datalake) UpdateServiceClient(key, value string) (err error)

UpdateServiceClient : Update the SAS specified by the user and create new service client

func (*Datalake) Write 

func (dl *Datalake) Write(options internal.WriteFileOptions) error

Write : Write to a file at given offset

func (*Datalake) WriteFromBuffer 

func (dl *Datalake) WriteFromBuffer(name string, metadata map[string]*string, data []byte) error

WriteFromBuffer : Upload from a buffer to a file

func (*Datalake) WriteFromFile 

func (dl *Datalake) WriteFromFile(name string, metadata map[string]*string, fi *os.File) (err error)

WriteFromFile : Upload local file to file

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.