Documentation ¶
Overview ¶
Provides a higher-level file access interface which is implemented using local file storage as well as AWS S3. This makes writing code that is agnostic to file storage medium much easier. Both are tested with the same unit testing framework to ensure they are compatible NOTE: In the FileAccess interface any reference to a bucket when used with the local file storage model is concatented with the path, so it could be thought of as a reference to the disk the path is on, or the root of where relative paths are available from
Example (GetBucketFromS3Url) ¶
b, err := GetBucketFromS3Url("some/path/file.json") fmt.Printf("%v|%v\n", b, err) b, err = GetBucketFromS3Url("s3:///path/file.json") fmt.Printf("%v|%v\n", b, err) b, err = GetBucketFromS3Url("s3://bucket") fmt.Printf("%v|%v\n", b, err) b, err = GetBucketFromS3Url("s3://the_bucket/some/path.json") fmt.Printf("%v|%v\n", b, err)
Output: |GetBucketFromS3Url parameter was not a valid S3 url: some/path/file.json |GetBucketFromS3Url failed to get bucket from S3 url: s3:///path/file.json |GetBucketFromS3Url failed to get bucket from S3 url: s3://bucket the_bucket|<nil>
Example (GetPathFromS3Url) ¶
p, err := GetPathFromS3Url("some/path/file.json") fmt.Printf("%v|%v\n", p, err) p, err = GetPathFromS3Url("s3:///path/file.json") fmt.Printf("%v|%v\n", p, err) p, err = GetPathFromS3Url("s3://bucket") fmt.Printf("%v|%v\n", p, err) p, err = GetPathFromS3Url("s3://the_bucket/some/path.json") fmt.Printf("%v|%v\n", p, err)
Output: |GetPathFromS3Url parameter was not a valid S3 url: some/path/file.json |GetPathFromS3Url failed to get path from S3 url: s3:///path/file.json |GetPathFromS3Url failed to get path from S3 url: s3://bucket some/path.json|<nil>
Example (IsValidObjectName) ¶
fmt.Println(IsValidObjectName("name")) fmt.Println(IsValidObjectName("Name With Spaces")) fmt.Println(IsValidObjectName("Name With Spaces")) fmt.Println(IsValidObjectName("")) fmt.Println(IsValidObjectName("Name \"Quote"))
Output: true true true false false
Example (LocalFileSystem) ¶
// First, clear any files we may have there already fmt.Printf("Setup: %v\n", os.RemoveAll("./test-output/")) // Now run the tests runTest(&FSAccess{}, "./test-output") // NOTE: test output must match the output from S3 (except cleanup steps)
Output: Setup: <nil> JSON: <nil> JSON no-indent: <nil> Exists1: false|<nil> Binary: <nil> Exists2: true|<nil> Copy: <nil> Copy bad path, got not found error: true Read JSON: <nil>, {Hello 778 World} Read JSON no-indent: <nil>, {Hello 778 World} Read Binary: <nil>, [250 130 10 0 33] Read bad path, got not found error: true Read bad JSON: invalid character 'ú' looking for beginning of value Not a "not found" error: true Listing: <nil>, [the-files/data.bin the-files/pretty.json the-files/subdir/copied.json the-files/subdir/ugly.json] Listing subdir: <nil>, [the-files/subdir/copied.json the-files/subdir/ugly.json] Listing with prefix: <nil>, [the-files/subdir/ugly.json] Listing bad path: <nil>, [] Delete copy: <nil> Delete bin: <nil> Listing2: <nil>, [the-files/pretty.json the-files/subdir/ugly.json] Listing subdir2: <nil>, [the-files/subdir/ugly.json] Empty dir: <nil> Listing subdir3: <nil>, []
Example (MakeValidObjectName) ¶
fmt.Println(MakeValidObjectName("my file!", true)) fmt.Println(MakeValidObjectName("this/path/to.bin", true)) fmt.Println(MakeValidObjectName("Hope \"this\" isn't too $expensive", true)) fmt.Println(MakeValidObjectName("This-file is it", true)) fmt.Println(MakeValidObjectName("A!B#C$D/E\\F", true)) fmt.Println(MakeValidObjectName("This-file; is it", true)) fmt.Println(MakeValidObjectName("This-file is it", true)) fmt.Println(MakeValidObjectName("This-file is it", false))
Output: my file this_path_to.bin Hope this isnt too expensive This-file is it ABCD_E_F This-file is it This-file is it This-file_is_it
Example (S3) ¶
rand.Seed(time.Now().UnixNano()) sess, err := awsutil.GetSessionWithRegion("us-east-1") if err != nil { fmt.Println("Failed to get AWS session") return } s3svc, err := awsutil.GetS3(sess) if err != nil { fmt.Println("Failed to get S3") return } fmt.Printf("Setup: %v\n", err) fs := MakeS3Access(s3svc) // Create test S3 bucket for this purpose testBucket := "api-fileaccess-s3-test-" + utils.RandStringBytesMaskImpr(10) _, err = s3svc.CreateBucket( &s3.CreateBucketInput{ Bucket: aws.String(testBucket), //CreateBucketConfiguration: }, ) if err != nil { fmt.Printf("Failed to create test S3 bucket: %v\n", err) return } defer func() { _, err := s3svc.DeleteBucket(&s3.DeleteBucketInput{Bucket: aws.String(testBucket)}) fmt.Printf("Delete bucket errors: %v\n", err) }() // Now run the tests runTest(fs, testBucket) // NOTE: test output must match the output from local file system (except cleanup steps)
Output: Setup: <nil> JSON: <nil> JSON no-indent: <nil> Exists1: false|<nil> Binary: <nil> Exists2: true|<nil> Copy: <nil> Copy bad path, got not found error: true Read JSON: <nil>, {Hello 778 World} Read JSON no-indent: <nil>, {Hello 778 World} Read Binary: <nil>, [250 130 10 0 33] Read bad path, got not found error: true Read bad JSON: invalid character 'ú' looking for beginning of value Not a "not found" error: true Listing: <nil>, [the-files/data.bin the-files/pretty.json the-files/subdir/copied.json the-files/subdir/ugly.json] Listing subdir: <nil>, [the-files/subdir/copied.json the-files/subdir/ugly.json] Listing with prefix: <nil>, [the-files/subdir/ugly.json] Listing bad path: <nil>, [] Delete copy: <nil> Delete bin: <nil> Listing2: <nil>, [the-files/pretty.json the-files/subdir/ugly.json] Listing subdir2: <nil>, [the-files/subdir/ugly.json] Empty dir: <nil> Listing subdir3: <nil>, [] Delete bucket errors: <nil>
Example (S3ListingWithContinuation) ¶
const bucket = "dev-pixlise-data" const listPath = "Datasets/" var mockS3 awsutil.MockS3Client defer mockS3.FinishTest() mockS3.ExpListObjectsV2Input = []s3.ListObjectsV2Input{ { Bucket: aws.String(bucket), Prefix: aws.String(listPath), }, { Bucket: aws.String(bucket), Prefix: aws.String(listPath), ContinuationToken: aws.String("cont-1"), }, { Bucket: aws.String(bucket), Prefix: aws.String(listPath), ContinuationToken: aws.String("cont-2"), }, } mockS3.QueuedListObjectsV2Output = []*s3.ListObjectsV2Output{ { IsTruncated: aws.Bool(true), NextContinuationToken: aws.String("cont-1"), Contents: []*s3.Object{ {Key: aws.String("Datasets/abc-123/summary.json")}, {Key: aws.String("Datasets/abc-123/node1.json")}, {Key: aws.String("Datasets/abc-123/params.json")}, }, }, { IsTruncated: aws.Bool(true), NextContinuationToken: aws.String("cont-2"), Contents: []*s3.Object{ {Key: aws.String("Datasets/abc-456/summary.json")}, {Key: aws.String("Datasets/abc-789/summary.json")}, {Key: aws.String("Datasets/")}, // Happens when we create a path in S3 web console, but has no use for us, so we filter it {Key: aws.String("Datasets/abc-456/params.json")}, }, }, { IsTruncated: aws.Bool(false), Contents: []*s3.Object{ {Key: aws.String("Datasets/abc-456/output/combined.csv")}, }, }, } fs := MakeS3Access(&mockS3) list, err := fs.ListObjects(bucket, listPath) fmt.Printf("%v, list: %v\n", err, list)
Output: <nil>, list: [Datasets/abc-123/summary.json Datasets/abc-123/node1.json Datasets/abc-123/params.json Datasets/abc-456/summary.json Datasets/abc-789/summary.json Datasets/abc-456/params.json Datasets/abc-456/output/combined.csv]
Index ¶
- func CopyFileLocally(srcPath string, dstPath string) error
- func GetBucketFromS3Url(url string) (string, error)
- func GetPathFromS3Url(url string) (string, error)
- func IsValidObjectName(name string) bool
- func MakeEmptyLocalDirectory(root string, subdir string) (string, error)
- func MakeValidObjectName(name string, allowSpace bool) string
- type FSAccess
- func (fs *FSAccess) CopyObject(srcRootPath string, srcPath string, dstRootPath string, dstPath string) error
- func (fs *FSAccess) DeleteObject(rootPath string, path string) error
- func (fs *FSAccess) EmptyObjects(rootPath string) error
- func (fs *FSAccess) IsNotFoundError(err error) bool
- func (fs *FSAccess) ListObjects(rootPath string, prefix string) ([]string, error)
- func (fs *FSAccess) ObjectExists(rootPath string, path string) (bool, error)
- func (fs *FSAccess) ReadJSON(rootPath string, s3Path string, itemsPtr interface{}, emptyIfNotFound bool) error
- func (fs *FSAccess) ReadObject(rootPath string, path string) ([]byte, error)
- func (fs *FSAccess) WriteJSON(rootPath string, s3Path string, itemsPtr interface{}) error
- func (fs *FSAccess) WriteJSONNoIndent(rootPath string, s3Path string, itemsPtr interface{}) error
- func (fs *FSAccess) WriteObject(rootPath string, path string, data []byte) error
- type FileAccess
- type Mock
- type S3Access
- func (s3Access S3Access) CopyObject(srcBucket string, srcPath string, dstBucket string, dstPath string) error
- func (s3Access S3Access) DeleteObject(bucket string, path string) error
- func (s3Access S3Access) EmptyObjects(targetBucket string) error
- func (s3Access S3Access) IsNotFoundError(err error) bool
- func (s3Access S3Access) ListObjects(bucket string, prefix string) ([]string, error)
- func (s3Access S3Access) ObjectExists(bucket string, path string) (bool, error)
- func (s3Access S3Access) ReadJSON(bucket string, s3Path string, itemsPtr interface{}, emptyIfNotFound bool) error
- func (s3Access S3Access) ReadObject(bucket string, path string) ([]byte, error)
- func (s3Access S3Access) WriteJSON(bucket string, s3Path string, itemsPtr interface{}) error
- func (s3Access S3Access) WriteJSONNoIndent(bucket string, s3Path string, itemsPtr interface{}) error
- func (s3Access S3Access) WriteObject(bucket string, path string, data []byte) error
Examples ¶
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
func CopyFileLocally ¶
func GetBucketFromS3Url ¶
func GetPathFromS3Url ¶
func IsValidObjectName ¶
Is this string a valid name to use as an AWS object name?
func MakeEmptyLocalDirectory ¶
Creates a directory under the specified root, ensures it's empty (eg if it already existed)
func MakeValidObjectName ¶
Turns a string name potentially typed by a user into a file name that should be valid for storage in anything we store in. This removes or replaces illegal characters with _.
Types ¶
type FSAccess ¶
type FSAccess struct { }
Implementation of file access using local file system
func (*FSAccess) CopyObject ¶
func (*FSAccess) DeleteObject ¶
func (*FSAccess) EmptyObjects ¶
func (*FSAccess) IsNotFoundError ¶
func (*FSAccess) ListObjects ¶
func (*FSAccess) ObjectExists ¶
func (*FSAccess) ReadObject ¶
func (*FSAccess) WriteJSONNoIndent ¶
type FileAccess ¶
type FileAccess interface { // Effectively recursive listing of files in a given directory ListObjects(bucket string, prefix string) ([]string, error) // Does a file at a given path exist ObjectExists(rootPath string, path string) (bool, error) // Reads a file as bytes ReadObject(bucket string, path string) ([]byte, error) // Writes a file as bytes WriteObject(bucket string, path string, data []byte) error // Reads a file as JSON and decodes it into itemsPtr ReadJSON(bucket string, s3Path string, itemsPtr interface{}, emptyIfNotFound bool) error // Writes itemsPtr as a JSON file WriteJSON(bucket string, s3Path string, itemsPtr interface{}) error // Same as WriteJSON, but a few places in the code need to write // non-pretty-printed JSON for those files to work with Athena queries for // example. Instead of adding a flag to WriteJSON this is easier to implement. // Searching for WriteJSON still returns these! WriteJSONNoIndent(bucket string, s3Path string, itemsPtr interface{}) error // Delete a file DeleteObject(bucket string, path string) error // Copy a file CopyObject(srcBucket string, srcPath string, dstBucket string, dstPath string) error // Effectively performs "rm -rf" of all files the given bucket/root directory EmptyObjects(targetBucket string) error // Checks if the given error is a "not found" error for the implementation. This is because // AWS S3 would provide a different "not found" error than would a local file system fopen() failing IsNotFoundError(err error) bool }
type S3Access ¶
type S3Access struct {
// contains filtered or unexported fields
}
Implementation of file access using AWS S3
func MakeS3Access ¶
func (S3Access) CopyObject ¶
func (S3Access) DeleteObject ¶
func (S3Access) EmptyObjects ¶
func (S3Access) IsNotFoundError ¶
func (S3Access) ListObjects ¶
ListObjects - calls AWS ListObjectsV2 and if a continuation token is returned this keeps looping and storing more items until no more continuation tokens are left.