common

package
v0.33.3 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jan 23, 2025 License: GPL-3.0 Imports: 39 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	URIPrefix = "s3://"

	MIN_CHUNK_SIZE = 1024 * 1024 * 200
	MAX_CHUNK_SIZE = 1024 * 1024 * 1024 * 5

	ApiLimitMaxItems = 1000

	MaxBatchSize = 1000

	MinBatchSize = 1

	HeadContentLengthBase = 10

	HeadContentLengthBitSize = 64
)
View Source 
const (
	ObjectLockModeGovernance = ObjectLockMode("GOVERNANCE")
	ObjectLockModeCompliance = ObjectLockMode("COMPLIANCE")
)

Variables

View Source 
var DefaultObjectLockingBody = ObjectLockingBody{
	ObjectLockEnabled: "Enabled",
	Namespace:         namespace,
	Rule: ObjectLockRule{
		DefaultRetention: ObjectLockRuleDefaultRetention{
			Mode: ObjectLockModeCompliance,
		},
	},
}

Functions

func ApplyFilters 

func ApplyFilters(ctx context.Context, entries <-chan pipeline.WalkDirEntry, params []FilterParams, cancel context.CancelCauseFunc) <-chan pipeline.WalkDirEntry

func BuildBucketHostURL 

func BuildBucketHostURL(cfg Config, bucketName BucketName) (*url.URL, error)

func BuildBucketHostWithPathURL 

func BuildBucketHostWithPathURL(cfg Config, bucketName BucketName, path string) (*url.URL, error)

func BuildHostURL 

func BuildHostURL(cfg Config) (*url.URL, error)

func CopyMultipleFiles 

func CopyMultipleFiles(ctx context.Context, cfg Config, params CopyAllObjectsParams) error

func CopySingleFile 

func CopySingleFile(ctx context.Context, cfg Config, src mgcSchemaPkg.URI, dst mgcSchemaPkg.URI, storageClass string) error

func Delete 

func Delete(ctx context.Context, params DeleteObjectParams, cfg Config) error

func DeleteAllObjectsInBucket 

func DeleteAllObjectsInBucket(ctx context.Context, params DeleteAllObjectsInBucketParams, cfg Config) error

func DeleteBucket 

func DeleteBucket(ctx context.Context, params DeleteBucketParams, cfg Config) error

func DeleteObjects 

func DeleteObjects(ctx context.Context, params DeleteObjectsParams, cfg Config) error

func DeleteSingle 

func DeleteSingle(ctx context.Context, params DeleteObjectParams, cfg Config) error

func ExtractErr 

func ExtractErr(resp *http.Response, req *http.Request) error

func GetAbsSystemURI 

func GetAbsSystemURI(uri mgcSchemaPkg.URI) (mgcSchemaPkg.URI, error)

func GetDownloadDirDst 

func GetDownloadDirDst(dst mgcSchemaPkg.FilePath, src mgcSchemaPkg.URI) (mgcSchemaPkg.FilePath, error)

func GetDownloadFileDst 

func GetDownloadFileDst(dst mgcSchemaPkg.FilePath, src mgcSchemaPkg.URI) (mgcSchemaPkg.FilePath, error)

Same as GetDownloadDst, but if resulting 'dst' is a directory, append filename from 'src'

func ListGenerator 

func ListGenerator(ctx context.Context, params ListObjectsParams, cfg Config, onNewPage func(objCount uint64)) (outputChan <-chan pipeline.WalkDirEntry)

func NewCompletionPart 

func NewCompletionPart(partNumber int, etag string) completionPart

func NewCopier 

func NewCopier(ctx context.Context, cfg Config, src mgcSchemaPkg.URI, dst mgcSchemaPkg.URI, version string, storageClass string) (copier, error)

func NewDownloadRequest 

func NewDownloadRequest(ctx context.Context, cfg Config, src mgcSchemaPkg.URI, version string) (*http.Request, error)

func NewDownloader 

func NewDownloader(ctx context.Context, cfg Config, src mgcSchemaPkg.URI, dst mgcSchemaPkg.FilePath, version string) (downloader, error)

func NewUploader 

func NewUploader(cfg Config, src mgcSchemaPkg.FilePath, dst mgcSchemaPkg.URI, storageClass string) (uploader, error)

func SendRequest 

func SendRequest(ctx context.Context, req *http.Request) (res *http.Response, err error)

func SendRequestWithIgnoredHeaders 

func SendRequestWithIgnoredHeaders(ctx context.Context, req *http.Request, ignoredHeaders map[string]struct{}) (res *http.Response, err error)

func SignedUrl 

func SignedUrl(req *http.Request, accessKey, secretKey string, expirationTime time.Duration) (url *url.URL, err error)

func UnwrapResponse 

func UnwrapResponse[T any](resp *http.Response, req *http.Request) (result T, err error)

func WriteToFile 

func WriteToFile(ctx context.Context, reader io.ReadCloser, fileSize int64, outFile mgcSchemaPkg.FilePath) (err error)

Types

type ACLCannedPermissions 

type ACLCannedPermissions struct {
	Private           bool `` /* 133-byte string literal not displayed */
	PublicRead        bool `json:"public_read,omitempty" jsonschema:"description=Owner gets FULL_CONTROL. Everyone else has READ rights"`
	PublicReadWrite   bool `` /* 137-byte string literal not displayed */
	AuthenticatedRead bool `` /* 135-byte string literal not displayed */
	AwsExecRead       bool `json:"aws_exec_read,omitempty" mgc:"hidden"`
}

func (ACLCannedPermissions) IsEmpty 

func (o ACLCannedPermissions) IsEmpty() bool

func (ACLCannedPermissions) SetHeaders 

func (o ACLCannedPermissions) SetHeaders(req *http.Request)

func (ACLCannedPermissions) Validate 

func (o ACLCannedPermissions) Validate() error

type ACLPermission 

type ACLPermission struct {
	ID string `json:"id" jsonschema:"description=Either a Tenant ID or a User Project ID,example=a4900b57-7dbb-4906-b7e8-efed938e325c"`
}

func (ACLPermission) Validate 

func (g ACLPermission) Validate() error

type ACLPermissions 

type ACLPermissions struct {
	ACLStandardPermissions `json:",squash"` // nolint
	ACLCannedPermissions   `json:",squash"` // nolint
}

func (ACLPermissions) IsEmpty 

func (p ACLPermissions) IsEmpty() bool

func (ACLPermissions) SetHeaders 

func (p ACLPermissions) SetHeaders(req *http.Request, cfg Config) error

func (ACLPermissions) Validate 

func (p ACLPermissions) Validate() error

type ACLStandardPermissions 

type ACLStandardPermissions struct {
	GrantFullControl []ACLPermission `json:"grant_full_control,omitempty" jsonschema:"description=Grantees get FULL_CONTROL" mgc:"hidden"`
	GrantRead        []ACLPermission `json:"grant_read,omitempty" jsonschema:"description=Allows grantees to list the objects in the bucket" mgc:"hidden"`
	GrantWrite       []ACLPermission `json:"grant_write,omitempty" jsonschema:"description=Allows grantees to create objects in the bucket"`
	GrantReadAcp     []ACLPermission `json:"grant_read_acp,omitempty" jsonschema:"description=Allows grantees to read the bucket ACL" mgc:"hidden"`
	GrantWriteAcp    []ACLPermission `` /* 129-byte string literal not displayed */
}

func (ACLStandardPermissions) IsEmpty 

func (p ACLStandardPermissions) IsEmpty() bool

func (ACLStandardPermissions) SetHeaders 

func (p ACLStandardPermissions) SetHeaders(req *http.Request, cfg Config) (err error)

func (ACLStandardPermissions) Validate 

func (o ACLStandardPermissions) Validate() error

type AccessControlList 

type AccessControlList struct {
	Grant []Grant `xml:"Grant"`
}

type AccessControlPolicy 

type AccessControlPolicy struct {
	Owner             Owner             `xml:"Owner"`
	AccessControlList AccessControlList `xml:"AccessControlList"`

	XMLName struct{} `xml:"AccessControlPolicy" json:"-"`
}

type BucketContent 

type BucketContent struct {
	Key          string `xml:"Key"`
	LastModified string `xml:"LastModified"`
	ContentSize  int64  `xml:"Size"`
	StorageClass string `xml:"StorageClass"`
}

func (*BucketContent) Info 

func (b *BucketContent) Info() (fs.FileInfo, error)

func (*BucketContent) IsDir 

func (b *BucketContent) IsDir() bool

func (*BucketContent) ModTime 

func (b *BucketContent) ModTime() time.Time

func (*BucketContent) Mode 

func (b *BucketContent) Mode() fs.FileMode

func (*BucketContent) Name 

func (b *BucketContent) Name() string

func (*BucketContent) Size 

func (b *BucketContent) Size() int64

func (*BucketContent) Sys 

func (b *BucketContent) Sys() any

func (*BucketContent) Type 

func (b *BucketContent) Type() fs.FileMode

type BucketContentDirEntry 

type BucketContentDirEntry = *pipeline.SimpleWalkDirEntry[*BucketContent]

type BucketHostString 

type BucketHostString string

func BuildBucketHost 

func BuildBucketHost(cfg Config, bucketName BucketName) (BucketHostString, error)

func BuildBucketHostWithPath 

func BuildBucketHostWithPath(cfg Config, bucketName BucketName, path string) (BucketHostString, error)

type BucketName 

type BucketName string

func NewBucketNameFromURI 

func NewBucketNameFromURI(u mgcSchemaPkg.URI) BucketName

func (BucketName) AsURI 

func (b BucketName) AsURI() mgcSchemaPkg.URI

func (BucketName) JSONSchemaExtend 

func (b BucketName) JSONSchemaExtend(s *jsonschema.Schema)

func (BucketName) String 

func (b BucketName) String() string

type Config 

type Config struct {
	Workers   int    `` /* 147-byte string literal not displayed */
	ChunkSize uint64 `` /* 154-byte string literal not displayed */
	Region    string `` /* 128-byte string literal not displayed */
	// See more about the 'squash' directive here: https://pkg.go.dev/github.com/mitchellh/mapstructure#hdr-Embedded_Structs_and_Squashing
	config.NetworkConfig `json:",squash"` // nolint
}

type CopyAllObjectsParams 

type CopyAllObjectsParams struct {
	Source       mgcSchemaPkg.URI `json:"src" jsonschema:"description=Path of objects in a bucket to be copied,example=bucket1" mgc:"positional"`
	Destination  mgcSchemaPkg.URI `json:"dst" jsonschema:"description=Full destination path in the bucket,example=bucket2/dir/" mgc:"positional"`
	StorageClass string           `` /* 179-byte string literal not displayed */
	Filters      `json:",squash"` // nolint
}

type CopyObjectParams 

type CopyObjectParams struct {
	Source       mgcSchemaPkg.URI `json:"src" jsonschema:"description=Path of the object in a bucket to be copied,example=bucket1/file.txt" mgc:"positional"`
	Destination  mgcSchemaPkg.URI `` /* 139-byte string literal not displayed */
	Version      string           `json:"obj_version,omitempty" jsonschema:"description=Version of the object to be copied"`
	StorageClass string           `` /* 179-byte string literal not displayed */
}

type DeleteAllObjectsInBucketParams 

type DeleteAllObjectsInBucketParams struct {
	BucketName BucketName       `json:"bucket" jsonschema:"description=Name of the bucket to delete objects from" mgc:"positional"`
	BatchSize  int              `` /* 138-byte string literal not displayed */
	Filters    `json:",squash"` // nolint
}

type DeleteBucketParams 

type DeleteBucketParams struct {
	Destination mgcSchemaPkg.URI `json:"dst" jsonschema:"description=Path of the bucket to be deleted,example=bucket1" mgc:"positional"`
}

type DeleteObjectParams 

type DeleteObjectParams struct {
	Destination mgcSchemaPkg.URI `json:"dst" jsonschema:"description=Path of the object to be deleted,example=bucket1/file.txt" mgc:"positional"`
	Version     string           `json:"objVersion,omitempty" jsonschema:"description=Version of the object to be deleted"`
}

type DeleteObjectsParams 

type DeleteObjectsParams struct {
	Destination mgcSchemaPkg.URI
	ToDelete    <-chan pipeline.WalkDirEntry
	BatchSize   int
}

type DownloadObjectParams 

type DownloadObjectParams struct {
	Source      mgcSchemaPkg.URI      `json:"src" jsonschema:"description=Path of the object to be downloaded,example=bucket1/file.txt" mgc:"positional"`
	Destination mgcSchemaPkg.FilePath `` /* 195-byte string literal not displayed */
	Version     string                `json:"obj_version,omitempty" jsonschema:"description=Version of the object to be downloaded"`
}

type FilterParams 

type FilterParams struct {
	Include string `json:"include,omitempty" jsonschema:"description=Filename pattern to include"`
	Exclude string `json:"exclude,omitempty" jsonschema:"description=Filename pattern to exclude"`
}

type Filters 

type Filters struct {
	FilterParams []FilterParams `json:"filter,omitempty" jsonschema:"description=File name pattern to include or exclude"`
}

type Grant 

type Grant struct {
	Grantee    Grantee `xml:"Grantee"`
	Permission string  `xml:"Permission"`
}

type Grantee 

type Grantee struct {
	DisplayName string `xml:"DisplayName"`
	// EmailAddress string `xml:"EmailAddress"`
	ID  string `xml:"ID"`
	URI string `xml:"URI"`
}

type HeadObjectResponse 

type HeadObjectResponse struct {
	AcceptRanges  string
	LastModified  string
	ContentLength int64
	ETag          string
	ContentType   string
	StorageClass  string
}

func HeadFile 

func HeadFile(ctx context.Context, cfg Config, dst mgcSchemaPkg.URI, version string) (metadata HeadObjectResponse, err error)

type HeaderMap 

type HeaderMap = map[string]any

type HostString 

type HostString string

func BuildHost 

func BuildHost(cfg Config) HostString

type ListObjectsParams 

type ListObjectsParams struct {
	Destination      mgcSchemaPkg.URI `json:"dst" jsonschema:"description=Path of the bucket to list objects from,example=bucket1" mgc:"positional"`
	PaginationParams `json:",squash"` // nolint
	Recursive        bool             `json:"recursive,omitempty" jsonschema:"description=List folders and subfolders,default=false"`
}

type ObjectError 

type ObjectError struct {
	Url mgcSchemaPkg.URI
	Err error
}

func (*ObjectError) Error 

func (e *ObjectError) Error() string

func (*ObjectError) Unwrap 

func (e *ObjectError) Unwrap() error

type ObjectLockMode 

type ObjectLockMode string

type ObjectLockRule 

type ObjectLockRule struct {
	DefaultRetention ObjectLockRuleDefaultRetention
}

type ObjectLockRuleDefaultRetention 

type ObjectLockRuleDefaultRetention struct {
	Days  int `xml:",omitempty"`
	Mode  ObjectLockMode
	Years int `xml:",omitempty"`
}

Object lock [Bucket]

type ObjectLockingBody 

type ObjectLockingBody struct {
	XMLName           xml.Name `xml:"ObjectLockConfiguration,omitempty"`
	Namespace         string   `xml:"xmlns,attr"`
	ObjectLockEnabled string
	Rule              ObjectLockRule
}

type ObjectRetention 

type ObjectRetention struct {
	XMLName         xml.Name `xml:"Retention"`
	Namespace       string   `xml:"xmlns,attr"`
	Mode            ObjectLockMode
	RetainUntilDate string `xml:",omitempty"`
}

Object retention [Object]

func DefaultObjectRetentionBody 

func DefaultObjectRetentionBody(retainUntilDate time.Time) ObjectRetention

type Owner 

type Owner struct {
	DisplayName string `xml:"DisplayName"`
	ID          string `xml:"ID" type:"ID"`
}

Container for the owner's display name and ID.

type PaginationParams 

type PaginationParams struct {
	MaxItems          int    `json:"max-items,omitempty" jsonschema:"description=Limit of items to be listed,default=1000,minimum=1,example=1000"`
	ContinuationToken string `json:"continuation-token,omitempty" jsonschema:"description=Token of result page to continue from"`
}

type Prefix 

type Prefix struct {
	Path string `xml:"Prefix"`
}

func (*Prefix) Info 

func (p *Prefix) Info() (fs.FileInfo, error)

func (*Prefix) IsDir 

func (p *Prefix) IsDir() bool

func (*Prefix) ModTime 

func (p *Prefix) ModTime() time.Time

func (*Prefix) Mode 

func (p *Prefix) Mode() fs.FileMode

func (*Prefix) Name 

func (p *Prefix) Name() string

func (*Prefix) Size 

func (p *Prefix) Size() int64

func (*Prefix) Sys 

func (p *Prefix) Sys() any

func (*Prefix) Type 

func (p *Prefix) Type() fs.FileMode

type PublicUrlResult 

type PublicUrlResult struct {
	URL mgcSchemaPkg.URI `json:"url"`
}

func PublicUrl 

func PublicUrl(ctx context.Context, cfg Config, dst mgcSchemaPkg.URI) (url *PublicUrlResult, err error)

type SignatureContext 

type SignatureContext struct {
	Parameters SignatureParameters

	// Request
	HTTPMethod       string
	CanonicalURI     string
	CanonicalQuery   string
	CanonicalHeaders string

	SignedHeaders string

	// these are set by Sign
	Signature string
}

func NewSignatureContext 

func NewSignatureContext(
	params SignatureParameters,
	req *http.Request,
) *SignatureContext

type SignatureParameters 

type SignatureParameters struct {
	Algorithm     string
	AccessKey     string
	Credential    string
	Scope         string
	Date          string
	ShortDate     string
	PayloadHash   string
	SignedHeaders []string // must be lower-case and sorted, see https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html
}

func NewSignatureParameters 

func NewSignatureParameters(
	accessKey string,
	signingTime time.Time,
	payloadHash string,
	signedHeaders []string,
) SignatureParameters

type XMLError 

type XMLError struct {
	Message string `xml:"Message"`
	Code    string `xml:"Code"`
}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.