s3fs

README

Package cloudeng.io/aws/s3fs

import cloudeng.io/aws/s3fs

Package s3fs implements fs.FS for AWS S3.

Package s3fs implements fs.FS for AWS S3.

Functions

Func New

func New(cfg aws.Config, options ...Option) file.FS

New creates a new instance of file.FS backed by S3.

Types

Type Client

type Client interface {
	GetObject(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.Options)) (*s3.GetObjectOutput, error)
}

Client represents the set of AWS S3 client methods used by s3fs.

Type Factory

type Factory struct {
	Config awsconfig.AWSFlags
}

Factory implements file.FSFactory for AWS S3.

Methods

func (d Factory) New(ctx context.Context, _ string) (file.FS, error)

New implements file.FSFactory.

func (d Factory) NewFromMatch(ctx context.Context, match cloudpath.Match) (file.FS, error)

Type Option

type Option func(o *options)

Option represents an option to New.

Functions

func WithS3Client(client Client) Option

WithS3Client specifies the s3.Client to use. If not specified, a new is created.

func WithS3Options(opts ...func(*s3.Options)) Option

WithS3Options wraps s3.Options for use when creating an s3.Client.

Documentation

Overview

Package s3fs implements fs.FS for AWS S3.

Package s3fs implements fs.FS for AWS S3.

Index

• func DirectoryBucketAZ(bucket string) string
• func IsDirectoryBucket(bucket string) bool
• func New(cfg aws.Config, options ...Option) filewalk.FS
• func NewCheckpointOperation(fs *T) checkpoint.Operation
• func NewLevelScanner(client Client, delimiter byte, path string) filewalk.LevelScanner
• type Client
• type Factory
  • func (f Factory) New(ctx context.Context) (*T, error)
• type Option
  • func WithDelimiter(d byte) Option
  • func WithS3Client(client Client) Option
  • func WithS3Options(opts ...func(*s3.Options)) Option
  • func WithScanSize(s int) Option
• type T
  • func NewS3FS(cfg aws.Config, options ...Option) *T
  • func (s3fs *T) Base(p string) string
  • func (s3fs *T) Delete(ctx context.Context, path string) error
  • func (s3fs *T) DeleteAll(ctx context.Context, path string) error
  • func (s3fs *T) EnsurePrefix(_ context.Context, path string, _ fs.FileMode) error
  • func (s3fs *T) Get(ctx context.Context, path string) ([]byte, error)
  • func (s3fs *T) IsNotExist(err error) bool
  • func (s3fs *T) IsPermissionError(err error) bool
  • func (s3fs *T) Join(components ...string) string
  • func (fs *T) LevelScanner(prefix string) filewalk.LevelScanner
  • func (s3fs *T) Lstat(ctx context.Context, path string) (file.Info, error)
  • func (s3fs *T) Open(name string) (fs.File, error)
  • func (s3fs *T) OpenCtx(ctx context.Context, name string) (fs.File, error)
  • func (s3fs *T) Put(ctx context.Context, path string, _ fs.FileMode, data []byte) error
  • func (s3fs *T) Readlink(_ context.Context, _ string) (string, error)
  • func (s3fs *T) Scheme() string
  • func (s3fs *T) Stat(ctx context.Context, name string) (file.Info, error)
  • func (s3fs *T) SysXAttr(existing any, merge file.XAttr) any
  • func (s3fs *T) XAttr(_ context.Context, _ string, info file.Info) (file.XAttr, error)

Functions

func DirectoryBucketAZ

func DirectoryBucketAZ(bucket string) string

func IsDirectoryBucket

func IsDirectoryBucket(bucket string) bool

func New

func New(cfg aws.Config, options ...Option) filewalk.FS

New creates a new instance of filewalk.FS backed by S3.

func NewCheckpointOperation

func NewCheckpointOperation(fs *T) checkpoint.Operation

NewCheckpointOperation returns a checkpoint.Operation that uses the S3.

func NewLevelScanner

func NewLevelScanner(client Client, delimiter byte, path string) filewalk.LevelScanner

Types

type Client

type Client interface {
	GetObject(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.Options)) (*s3.GetObjectOutput, error)
	HeadObject(context.Context, *s3.HeadObjectInput, ...func(*s3.Options)) (*s3.HeadObjectOutput, error)
	GetBucketAcl(ctx context.Context, params *s3.GetBucketAclInput, optFns ...func(*s3.Options)) (*s3.GetBucketAclOutput, error)
	ListObjectsV2(context.Context, *s3.ListObjectsV2Input, ...func(*s3.Options)) (*s3.ListObjectsV2Output, error)
	PutObject(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error)
	DeleteObject(ctx context.Context, params *s3.DeleteObjectInput, optFns ...func(*s3.Options)) (*s3.DeleteObjectOutput, error)
	DeleteObjects(ctx context.Context, params *s3.DeleteObjectsInput, optFns ...func(*s3.Options)) (*s3.DeleteObjectsOutput, error)
}

Client represents the set of AWS S3 client methods used by s3fs.

type Factory

type Factory struct {
	Config  awsconfig.AWSFlags
	Options []Option
}

Factory wraps creating an S3FS with the configuration required to correctly initialize it.

func (Factory) New

func (f Factory) New(ctx context.Context) (*T, error)

type Option

type Option func(o *options)

Option represents an option to New.

func WithDelimiter

func WithDelimiter(d byte) Option

WithDelimiter sets the delimiter to use when listing objects, the default is /.

func WithS3Client

func WithS3Client(client Client) Option

WithS3Client specifies the s3.Client to use. If not specified, a new is created.

func WithS3Options

func WithS3Options(opts ...func(*s3.Options)) Option

WithS3Options wraps s3.Options for use when creating an s3.Client.

func WithScanSize

func WithScanSize(s int) Option

WithScanSize sets the number of items to fetch in a single remote api invocation for operations such as DeleteAll which may require iterating over a range of objects.

type T

type T struct {
	// contains filtered or unexported fields
}

func NewS3FS

func NewS3FS(cfg aws.Config, options ...Option) *T

NewS3FS creates a new instance of filewalk.FS and file.ObjectFS backed by S3.

func (*T) Base

func (s3fs *T) Base(p string) string

func (*T) Delete

func (s3fs *T) Delete(ctx context.Context, path string) error

func (*T) DeleteAll

func (s3fs *T) DeleteAll(ctx context.Context, path string) error

func (*T) EnsurePrefix

func (s3fs *T) EnsurePrefix(_ context.Context, path string, _ fs.FileMode) error

func (*T) Get

func (s3fs *T) Get(ctx context.Context, path string) ([]byte, error)

func (*T) IsNotExist

func (s3fs *T) IsNotExist(err error) bool

func (*T) IsPermissionError

func (s3fs *T) IsPermissionError(err error) bool

func (*T) Join

func (s3fs *T) Join(components ...string) string

Join concatenates the supplied components ensuring to insert delimiters only when necessary, that is components ending or starting with / (or the currently configured delimiter) will not

func (*T) LevelScanner

func (fs *T) LevelScanner(prefix string) filewalk.LevelScanner

func (*T) Lstat

func (s3fs *T) Lstat(ctx context.Context, path string) (file.Info, error)

func (*T) Open

func (s3fs *T) Open(name string) (fs.File, error)

Open implements fs.FS.

func (*T) OpenCtx

func (s3fs *T) OpenCtx(ctx context.Context, name string) (fs.File, error)

OpenCtx implements file.FS.

func (*T) Put

func (s3fs *T) Put(ctx context.Context, path string, _ fs.FileMode, data []byte) error

func (*T) Readlink

func (s3fs *T) Readlink(_ context.Context, _ string) (string, error)

ReadLink is not implemented for S3, ie. returns file.ErrNotImplemented.

func (*T) Scheme

func (s3fs *T) Scheme() string

Scheme implements fs.FS.

func (*T) Stat

func (s3fs *T) Stat(ctx context.Context, name string) (file.Info, error)

Stat invokes a Head operation on objects only. If name ends in / (or the currently configured delimiter) it is considered to be a prefix and a file.Info is created that reflects that (ie IsDir() returns true).

func (*T) SysXAttr

func (s3fs *T) SysXAttr(existing any, merge file.XAttr) any

func (*T) XAttr

func (s3fs *T) XAttr(_ context.Context, _ string, info file.Info) (file.XAttr, error)