s3fs

README

s3fs

A simple S3-based file store implementation for Go.

Example:

package main

import (
	"fmt"
	"os"
	"time"

	"github.com/d5/go-s3fs"
)

func main() {
	// create client
	client := s3fs.NewClient(
		os.Getenv("S3_BUCKET"), // S3 bucket name
		nil,                  // use default client options
		s3fs.NewAWSSession(), // AWS session
		s3fs.NewAWSConfigWithStaticCredentialsAndRegion(
			os.Getenv("AWS_ACCESS_KEY_ID"),     // AWS access key
			os.Getenv("AWS_SECRET_ACCESS_KEY"), // AWS secret key
			os.Getenv("S3_BUCKET_REGION")))     // S3 bucket region

	// write a file
	err := client.Write(&s3fs.File{
		Path:        "test/foo/bar",
		Data:        []byte("hello world!"),
		ContentType: s3fs.StringPtr("text/plain"),
	})
	if err != nil {
		panic(err)
	}

	// read a file
	file, err := client.Read("test/foo/bar")
	if err != nil {
		panic(err)
	}
	fmt.Println(string(file.Data))

	// get a pre-signed URL for the file
	link, err := client.GenerateURLWithExpire("test/foo/bar", 10*time.Minute)
	if err != nil {
		panic(err)
	}
	fmt.Println(link)
}

To run tests, you need to create your own S3 bucket and set the following environment variables.

export AWS_ACCESS_KEY_ID="<your AWS access key>"
export AWS_SECRET_ACCESS_KEY="<your AWS secret key>"
export TEST_S3_BUCKET="<your S3 bucket name>"
export TEST_S3_BUCKET_REGION="<your S3 bucket region>"

# run tests
make test

Documentation

Index

• Constants
• Variables
• func NewAWSConfigWithStaticCredentials(awsAccessKey, awsSecretKey string) *aws.Config
• func NewAWSConfigWithStaticCredentialsAndRegion(awsAccessKey, awsSecretKey, awsRegion string) *aws.Config
• func NewAWSSession() *session.Session
• func StringPtr(v string) *string
• type Client
  • func NewClient(bucket string, clientOptions *ClientOptions, awsSession client.ConfigProvider, ...) *Client
  • func (c *Client) Delete(path string) error
  • func (c *Client) DeleteAll(pathPrefix string) error
  • func (c *Client) GenerateURLWithExpire(path string, expire time.Duration) (string, error)
  • func (c *Client) ListAll(pathPrefix string) ([]ListItem, error)
  • func (c *Client) Read(path string) (*File, error)
  • func (c *Client) Write(file *File) error
• type ClientOptions
• type DownloadOptions
• type File
• type ListItem
• type UploadOptions

Constants

const (
	S3ACLPrivate                = "private"
	S3ACLPublicRead             = "public-read"
	S3ACLPublicReadWrite        = "public-read-write"
	S3ACLAuthenticatedRead      = "authenticated-read"
	S3ACLAWSExecRead            = "aws-exec-read"
	S3ACLBucketOwnerRead        = "bucket-owner-read"
	S3ACLBucketOwnerFullControl = "bucket-owner-full-control"
)

const (
	S3StorageClassStandard          = "STANDARD"
	S3StorageClassReducedRedundancy = "REDUCED_REDUNDANCY"
	S3StorageClassStandardIA        = "STANDARD_IA"
)

const (
	S3EncryptionAES256 = "AES256"
	S3EncryptionAWSKMS = "aws:kms"
)

Variables

var (
	ErrFileNotFound = errors.New("file not found")
)

Functions

func NewAWSConfigWithStaticCredentials

func NewAWSConfigWithStaticCredentials(awsAccessKey, awsSecretKey string) *aws.Config

NewAWSConfigWithStaticCredentials returns AWS config with static credentials.

func NewAWSConfigWithStaticCredentialsAndRegion

func NewAWSConfigWithStaticCredentialsAndRegion(awsAccessKey, awsSecretKey, awsRegion string) *aws.Config

NewAWSConfigWithStaticCredentialsAndRegion returns AWS config with static credentials and region.

func NewAWSSession

func NewAWSSession() *session.Session

NewAWSSession returns a default AWS session.

func StringPtr

func StringPtr(v string) *string

StringPtr converts string to string pointer.

Types

type Client

type Client struct {
	// contains filtered or unexported fields
}

func NewClient

func NewClient(bucket string, clientOptions *ClientOptions, awsSession client.ConfigProvider, awsConfigs ...*aws.Config) *Client

NewClient returns a new file store client.

func (*Client) Delete

func (c *Client) Delete(path string) error

Delete deletes a file in path. Because the way S3 handles deletion (adding delete-marker for versioning), this function does not return any errors when the file does not exist.

func (*Client) DeleteAll

func (c *Client) DeleteAll(pathPrefix string) error

DeleteAll deletes all files that have pathPrefix as path prefix.

func (*Client) GenerateURLWithExpire

func (c *Client) GenerateURLWithExpire(path string, expire time.Duration) (string, error)

GenerateURLWithExpire creates a direct link (URL) for the file. Anyone with this link will be able to access the file (through HTTP GET) regardless of S3 bucket permission settings. The link created with this function will return a permission error after it expires.

func (*Client) ListAll

func (c *Client) ListAll(pathPrefix string) ([]ListItem, error)

ListAll returns all files and directories that has pathPrefix as path prefix.

func (*Client) Read

func (c *Client) Read(path string) (*File, error)

Read retrieves a file data and its meta data. It returns ErrFileNotFound error if there's no file in path.

func (*Client) Write

func (c *Client) Write(file *File) error

type ClientOptions

type ClientOptions struct {
	UploadOptions   *UploadOptions
	DownloadOptions *DownloadOptions
}

type DownloadOptions

type DownloadOptions struct {
	PartSize    *int64
	Concurrency *int
}

type File

type File struct {
	// File path in the storage.
	Path string

	// File data.
	Data []byte

	// A map of metadata to store with the file in S3.
	// This is optional.
	Metadata map[string]string

	// Standard MIME type describing the format of the file data.
	// This is optional, and, will be set to "binary/octet-stream" by default.
	ContentType *string

	// Canned ACL to apply to the file.
	// Possible values include
	//  - "private"
	//  - "public-read"
	//  - "public-read-write"
	//  - "authenticated-read"
	//  - "aws-exec-read"
	//  - "bucket-owner-read"
	//  - "bucket-owner-full-control"
	// This is optional.
	ACL *string

	// Server-side encryption algorithm used when storing file.
	// Possible values include "AES256", "aws:kms".
	// This is optional.
	Encryption *string

	// Type of storage to use for the file.
	// Possible values include "STANDARD", "REDUCED_REDUNDANCY", "STANDARD_IA".
	// This is optional, and, will be set to "STANDARD" by default.
	StorageClass *string
}

File represents a file stored in S3 bucket.

type ListItem

type ListItem struct {
	Path  string // file or directory path (directory path ends with slash)
	IsDir bool   // whether item is directory or not
}

ListItem represents a file or directory in ListAll result.

type UploadOptions

type UploadOptions struct {
	PartSize          *int64
	Concurrency       *int
	LeavePartsOnError *bool
	MaxUploadParts    *int
}