fileaccess

Documentation

Overview

Example (LocalFileSystem)

// First, clear any files we may have there already
fmt.Printf("Setup: %v\n", os.RemoveAll("./test-output/"))

// Now run the tests
runTest(&FSAccess{}, "./test-output")

// NOTE: test output must match the output from S3 (except cleanup steps)

Output:

Setup: <nil>
JSON: <nil>
JSON no-indent: <nil>
Exists1: false|<nil>
Binary: <nil>
Exists2: true|<nil>
Copy: <nil>
Copy bad path, got not found error: true
Read JSON: <nil>, {Hello 778 World}
Read JSON no-indent: <nil>, {Hello 778 World}
Read Binary: <nil>, [250 130 10 0 33]
Read bad path, got not found error: true
Read bad JSON: invalid character 'ú' looking for beginning of value
Not a "not found" error: true
Listing: <nil>, [the-files/data.bin the-files/pretty.json the-files/subdir/copied.json the-files/subdir/ugly.json]
Listing subdir: <nil>, [the-files/subdir/copied.json the-files/subdir/ugly.json]
Listing with prefix: <nil>, [the-files/subdir/ugly.json]
Listing bad path: <nil>, []
Delete copy: <nil>
Delete bin: <nil>
Listing2: <nil>, [the-files/pretty.json the-files/subdir/ugly.json]
Listing subdir2: <nil>, [the-files/subdir/ugly.json]
Empty dir: <nil>
Listing subdir3: <nil>, []

Example (S3)

rand.Seed(time.Now().UnixNano())
sess, err := awsutil.GetSessionWithRegion("us-east-1")
if err != nil {
	fmt.Println("Failed to get AWS session")
	return
}
s3svc, err := awsutil.GetS3(sess)
if err != nil {
	fmt.Println("Failed to get S3")
	return
}

fmt.Printf("Setup: %v\n", err)

fs := MakeS3Access(s3svc)

// Create test S3 bucket for this purpose
testBucket := "api-fileaccess-s3-test-" + utils.RandStringBytesMaskImpr(10)
_, err = s3svc.CreateBucket(
	&s3.CreateBucketInput{
		Bucket: aws.String(testBucket),
		//CreateBucketConfiguration:
	},
)
if err != nil {
	fmt.Printf("Failed to create test S3 bucket: %v\n", err)
	return
}

defer func() {
	_, err := s3svc.DeleteBucket(&s3.DeleteBucketInput{Bucket: aws.String(testBucket)})
	fmt.Printf("Delete bucket errors: %v\n", err)
}()

// Now run the tests
runTest(fs, testBucket)

// NOTE: test output must match the output from local file system (except cleanup steps)

Output:

Setup: <nil>
JSON: <nil>
JSON no-indent: <nil>
Exists1: false|<nil>
Binary: <nil>
Exists2: true|<nil>
Copy: <nil>
Copy bad path, got not found error: true
Read JSON: <nil>, {Hello 778 World}
Read JSON no-indent: <nil>, {Hello 778 World}
Read Binary: <nil>, [250 130 10 0 33]
Read bad path, got not found error: true
Read bad JSON: invalid character 'ú' looking for beginning of value
Not a "not found" error: true
Listing: <nil>, [the-files/data.bin the-files/pretty.json the-files/subdir/copied.json the-files/subdir/ugly.json]
Listing subdir: <nil>, [the-files/subdir/copied.json the-files/subdir/ugly.json]
Listing with prefix: <nil>, [the-files/subdir/ugly.json]
Listing bad path: <nil>, []
Delete copy: <nil>
Delete bin: <nil>
Listing2: <nil>, [the-files/pretty.json the-files/subdir/ugly.json]
Listing subdir2: <nil>, [the-files/subdir/ugly.json]
Empty dir: <nil>
Listing subdir3: <nil>, []
Delete bucket errors: <nil>

Index

• func CopyFileLocally(srcPath string, dstPath string) error
• func IsValidObjectName(name string) bool
• func MakeEmptyLocalDirectory(root string, subdir string) (string, error)
• func MakeValidObjectName(name string, allowSpace bool) string
• type FSAccess
  • func (fs *FSAccess) CopyObject(srcRootPath string, srcPath string, dstRootPath string, dstPath string) error
  • func (fs *FSAccess) DeleteObject(rootPath string, path string) error
  • func (fs *FSAccess) EmptyObjects(rootPath string) error
  • func (fs *FSAccess) IsNotFoundError(err error) bool
  • func (fs *FSAccess) ListObjects(rootPath string, prefix string) ([]string, error)
  • func (fs *FSAccess) ObjectExists(rootPath string, path string) (bool, error)
  • func (fs *FSAccess) ReadJSON(rootPath string, s3Path string, itemsPtr interface{}, emptyIfNotFound bool) error
  • func (fs *FSAccess) ReadObject(rootPath string, path string) ([]byte, error)
  • func (fs *FSAccess) WriteJSON(rootPath string, s3Path string, itemsPtr interface{}) error
  • func (fs *FSAccess) WriteJSONNoIndent(rootPath string, s3Path string, itemsPtr interface{}) error
  • func (fs *FSAccess) WriteObject(rootPath string, path string, data []byte) error
• type FileAccess
• type Mock
• type S3Access
  • func MakeS3Access(s3Api s3iface.S3API) S3Access
  • func (s3Access S3Access) CopyObject(srcBucket string, srcPath string, dstBucket string, dstPath string) error
  • func (s3Access S3Access) DeleteObject(bucket string, path string) error
  • func (s3Access S3Access) EmptyObjects(targetBucket string) error
  • func (s3Access S3Access) IsNotFoundError(err error) bool
  • func (s3Access S3Access) ListObjects(bucket string, prefix string) ([]string, error)
  • func (s3Access S3Access) ObjectExists(bucket string, path string) (bool, error)
  • func (s3Access S3Access) ReadJSON(bucket string, s3Path string, itemsPtr interface{}, emptyIfNotFound bool) error
  • func (s3Access S3Access) ReadObject(bucket string, path string) ([]byte, error)
  • func (s3Access S3Access) WriteJSON(bucket string, s3Path string, itemsPtr interface{}) error
  • func (s3Access S3Access) WriteJSONNoIndent(bucket string, s3Path string, itemsPtr interface{}) error
  • func (s3Access S3Access) WriteObject(bucket string, path string, data []byte) error

Examples

• Package (LocalFileSystem)
• Package (S3)

Functions

func CopyFileLocally

func CopyFileLocally(srcPath string, dstPath string) error

func IsValidObjectName

func IsValidObjectName(name string) bool

Is this string a valid name to use as an AWS object name?

func MakeEmptyLocalDirectory

func MakeEmptyLocalDirectory(root string, subdir string) (string, error)

Creates a directory under the specified root, ensures it's empty (eg if it already existed)

func MakeValidObjectName

func MakeValidObjectName(name string, allowSpace bool) string

Types

type FSAccess

type FSAccess struct {
}

Implementation of file access using local file system

func (*FSAccess) CopyObject

func (fs *FSAccess) CopyObject(srcRootPath string, srcPath string, dstRootPath string, dstPath string) error

func (*FSAccess) DeleteObject

func (fs *FSAccess) DeleteObject(rootPath string, path string) error

func (*FSAccess) EmptyObjects

func (fs *FSAccess) EmptyObjects(rootPath string) error

func (*FSAccess) IsNotFoundError

func (fs *FSAccess) IsNotFoundError(err error) bool

func (*FSAccess) ListObjects

func (fs *FSAccess) ListObjects(rootPath string, prefix string) ([]string, error)

func (*FSAccess) ObjectExists

func (fs *FSAccess) ObjectExists(rootPath string, path string) (bool, error)

func (*FSAccess) ReadJSON

func (fs *FSAccess) ReadJSON(rootPath string, s3Path string, itemsPtr interface{}, emptyIfNotFound bool) error

func (*FSAccess) ReadObject

func (fs *FSAccess) ReadObject(rootPath string, path string) ([]byte, error)

func (*FSAccess) WriteJSON

func (fs *FSAccess) WriteJSON(rootPath string, s3Path string, itemsPtr interface{}) error

func (*FSAccess) WriteJSONNoIndent

func (fs *FSAccess) WriteJSONNoIndent(rootPath string, s3Path string, itemsPtr interface{}) error

func (*FSAccess) WriteObject

func (fs *FSAccess) WriteObject(rootPath string, path string, data []byte) error

type FileAccess

type FileAccess interface {
	ListObjects(bucket string, prefix string) ([]string, error)

	ObjectExists(rootPath string, path string) (bool, error)

	ReadObject(bucket string, path string) ([]byte, error)
	WriteObject(bucket string, path string, data []byte) error

	ReadJSON(bucket string, s3Path string, itemsPtr interface{}, emptyIfNotFound bool) error
	WriteJSON(bucket string, s3Path string, itemsPtr interface{}) error
	// A few places in the code need to write non-pretty-printed JSON for those
	// files to work with Athena queries. Instead of adding a flag to WriteJSON
	// this is easier to implement. Searching for WriteJSON still returns these!
	WriteJSONNoIndent(bucket string, s3Path string, itemsPtr interface{}) error

	DeleteObject(bucket string, path string) error

	CopyObject(srcBucket string, srcPath string, dstBucket string, dstPath string) error

	EmptyObjects(targetBucket string) error

	IsNotFoundError(err error) bool
}

type Mock

type Mock struct {
}

Mockup of file access implementation for unit tests

type S3Access

type S3Access struct {
	// contains filtered or unexported fields
}

Implementation of file access using AWS S3

func MakeS3Access

func MakeS3Access(s3Api s3iface.S3API) S3Access

func (S3Access) CopyObject

func (s3Access S3Access) CopyObject(srcBucket string, srcPath string, dstBucket string, dstPath string) error

func (S3Access) DeleteObject

func (s3Access S3Access) DeleteObject(bucket string, path string) error

func (S3Access) EmptyObjects

func (s3Access S3Access) EmptyObjects(targetBucket string) error

func (S3Access) IsNotFoundError

func (s3Access S3Access) IsNotFoundError(err error) bool

func (S3Access) ListObjects

func (s3Access S3Access) ListObjects(bucket string, prefix string) ([]string, error)

ListObjects - calls AWS ListObjectsV2 and if a continuation token is returned this keeps looping and storing more items until no more continuation tokens are left.

func (S3Access) ObjectExists

func (s3Access S3Access) ObjectExists(bucket string, path string) (bool, error)

func (S3Access) ReadJSON

func (s3Access S3Access) ReadJSON(bucket string, s3Path string, itemsPtr interface{}, emptyIfNotFound bool) error

func (S3Access) ReadObject

func (s3Access S3Access) ReadObject(bucket string, path string) ([]byte, error)

func (S3Access) WriteJSON

func (s3Access S3Access) WriteJSON(bucket string, s3Path string, itemsPtr interface{}) error

func (S3Access) WriteJSONNoIndent

func (s3Access S3Access) WriteJSONNoIndent(bucket string, s3Path string, itemsPtr interface{}) error

func (S3Access) WriteObject

func (s3Access S3Access) WriteObject(bucket string, path string, data []byte) error