filestore

README

filestore

Overview

filestore is a generic implementation for working with local files or cloud-based objects through a single interface. Currently, filestore supports AWS S3, with intentions of supporting Azure Blob Storage in the future.

Installation

go get github.com/USACE/filestore

Usage

package main

import "github.com/USACE/filestore"

func main() {

	config := filestore.S3FSConfig{
		S3Id:     "************",  // AWS_ACCESS_KEY_ID
		S3Key:    "************",  // AWS_SECRET_ACCESS_KEY
		S3Region: "us-east-1",
		S3Bucket: "my-bucket",
	}

	fs, err := filestore.NewFileStore(config)
	if err != nil {
		log.Fatal(err)
	}

	err = fs.Walk("path-prefix", walkFunc)
	if err != nil {
		log.Fatal(err)
	}

}

func walkFunc(path string, file os.FileInfo) error {
	fmt.Println(path)
	return nil
}

Documentation

Index

• type BlockFS
  • func (b *BlockFS) CompleteObjectUpload(u CompletedObjectUploadConfig) error
  • func (b *BlockFS) DeleteObjects(path ...string) error
  • func (b *BlockFS) GetDir(path string, recursive bool) (*[]FileStoreResultObject, error)
  • func (b *BlockFS) GetObject(path string) (io.ReadCloser, error)
  • func (b *BlockFS) InitializeObjectUpload(u UploadConfig) (UploadResult, error)
  • func (b *BlockFS) PutObject(path string, data []byte) (*FileOperationOutput, error)
  • func (b *BlockFS) Walk(path string, vistorFunction FileVisitFunction) error
  • func (b *BlockFS) WriteChunk(u UploadConfig) (UploadResult, error)
• type BlockFSConfig
• type CompletedObjectUploadConfig
• type FileOperationOutput
• type FileStore
  • func NewFileStore(config interface{}) (FileStore, error)
• type FileStoreResultObject
• type FileVisitFunction
• type PATHTYPE
• type PathParts
  • func (pp PathParts) ToFilePath(additionalParts ...string) string
  • func (pp PathParts) ToPath(additionalParts ...string) string
• type S3FS
  • func (s3fs *S3FS) CompleteObjectUpload(u CompletedObjectUploadConfig) error
  • func (s3fs *S3FS) DeleteObjects(path ...string) error
  • func (s3fs *S3FS) GetDir(path string, recursive bool) (*[]FileStoreResultObject, error)
  • func (s3fs *S3FS) GetObject(path string) (io.ReadCloser, error)
  • func (s3fs *S3FS) InitializeObjectUpload(u UploadConfig) (UploadResult, error)
  • func (s3fs *S3FS) Ping() error
  • func (s3fs *S3FS) PutObject(path string, data []byte) (*FileOperationOutput, error)
  • func (s3fs *S3FS) SetObjectPublic(path string) (string, error)
  • func (s3fs *S3FS) SharedAccessURL(path string, expiration time.Duration) (string, error)
  • func (s3fs *S3FS) Walk(path string, vistorFunction FileVisitFunction) error
  • func (s3fs *S3FS) WriteChunk(u UploadConfig) (UploadResult, error)
• type S3FSConfig
• type S3FileInfo
  • func (obj *S3FileInfo) IsDir() bool
  • func (obj *S3FileInfo) ModTime() time.Time
  • func (obj *S3FileInfo) Mode() os.FileMode
  • func (obj *S3FileInfo) Name() string
  • func (obj *S3FileInfo) Size() int64
  • func (obj *S3FileInfo) Sys() interface{}
• type UploadConfig
• type UploadResult

Types

type BlockFS

type BlockFS struct{}

func (*BlockFS) CompleteObjectUpload

func (b *BlockFS) CompleteObjectUpload(u CompletedObjectUploadConfig) error

func (*BlockFS) DeleteObjects

func (b *BlockFS) DeleteObjects(path ...string) error

func (*BlockFS) GetDir

func (b *BlockFS) GetDir(path string, recursive bool) (*[]FileStoreResultObject, error)

func (*BlockFS) GetObject

func (b *BlockFS) GetObject(path string) (io.ReadCloser, error)

func (*BlockFS) InitializeObjectUpload

func (b *BlockFS) InitializeObjectUpload(u UploadConfig) (UploadResult, error)

func (*BlockFS) PutObject

func (b *BlockFS) PutObject(path string, data []byte) (*FileOperationOutput, error)

func (*BlockFS) Walk

func (b *BlockFS) Walk(path string, vistorFunction FileVisitFunction) error

func (*BlockFS) WriteChunk

func (b *BlockFS) WriteChunk(u UploadConfig) (UploadResult, error)

type BlockFSConfig

type BlockFSConfig struct{}

@TODO this is kind of clunky. BlockFSConfig is only used in NewFileStore as a type case so we know to create a Block File Store as of now I don't actually need any config properties

type CompletedObjectUploadConfig

type CompletedObjectUploadConfig struct {
	UploadId string
	//PathInfo       models.ModelPathInfo
	//DirPath        string
	//FilePath       string
	ObjectPath string
	//ObjectName     string
	ChunkUploadIds []string
}

type FileOperationOutput

type FileOperationOutput struct {
	Md5 string
}

type FileStore

type FileStore interface {
	GetDir(string, bool) (*[]FileStoreResultObject, error)
	GetObject(string) (io.ReadCloser, error)
	PutObject(string, []byte) (*FileOperationOutput, error)
	DeleteObjects(path ...string) error
	//PutMultipartObject(u UploadConfig) (UploadResult, error)
	//InitializeMultipartWrite
	//PutPart(u UploadConfig) (UploadResult, error)
	Walk(string, FileVisitFunction) error

	/////depricate
	InitializeObjectUpload(UploadConfig) (UploadResult, error)
	WriteChunk(UploadConfig) (UploadResult, error)
	CompleteObjectUpload(CompletedObjectUploadConfig) error
}

func NewFileStore

func NewFileStore(config interface{}) (FileStore, error)

type FileStoreResultObject

type FileStoreResultObject struct {
	ID         int       `json:"id"`
	Name       string    `json:"fileName"`
	Size       string    `json:"size"`
	Path       string    `json:"filePath"`
	Type       string    `json:"type"`
	IsDir      bool      `json:"isdir"`
	Modified   time.Time `json:"modified"`
	ModifiedBy string    `json:"modifiedBy"`
}

type FileVisitFunction

type FileVisitFunction func(path string, file os.FileInfo) error

type PATHTYPE

type PATHTYPE int

const (
	FILE PATHTYPE = iota
	FOLDER
)

type PathParts

type PathParts struct {
	Parts []string
}

func (PathParts) ToFilePath

func (pp PathParts) ToFilePath(additionalParts ...string) string

func (PathParts) ToPath

func (pp PathParts) ToPath(additionalParts ...string) string

type S3FS

type S3FS struct {
	// contains filtered or unexported fields
}

S3FS satisfies the FileStore interface, allowing for generic file operations to be done on s3 blobs

func (*S3FS) CompleteObjectUpload

func (s3fs *S3FS) CompleteObjectUpload(u CompletedObjectUploadConfig) error

func (*S3FS) DeleteObjects

func (s3fs *S3FS) DeleteObjects(path ...string) error

DeleteObjects will take one or more paths, and delete them from the s3 file system

func (*S3FS) GetDir

func (s3fs *S3FS) GetDir(path string, recursive bool) (*[]FileStoreResultObject, error)

GetDir is similar to an ls unix call. It lists the objects at an s3 prefix, with the option of being recursive

func (*S3FS) GetObject

func (s3fs *S3FS) GetObject(path string) (io.ReadCloser, error)

GetObject will return the body of an s3 object as a ReadCloser, meaning it has the basic Read and Close methods

func (*S3FS) InitializeObjectUpload

func (s3fs *S3FS) InitializeObjectUpload(u UploadConfig) (UploadResult, error)

func (*S3FS) Ping

func (s3fs *S3FS) Ping() error

Ping makes a cheap call to the s3 bucket to ensure connection

func (*S3FS) PutObject

func (s3fs *S3FS) PutObject(path string, data []byte) (*FileOperationOutput, error)

PutObject will take the data provided and put it on s3 at the path provided

func (*S3FS) SetObjectPublic

func (s3fs *S3FS) SetObjectPublic(path string) (string, error)

SetObjectPublic will change the acl permissions on an s3 object and make it publically readable

func (*S3FS) SharedAccessURL

func (s3fs *S3FS) SharedAccessURL(path string, expiration time.Duration) (string, error)

SharedAccessURL will create a presigned url that can be used to access/download an object from an s3 bucket. It will only be valid for the duration specified

func (*S3FS) Walk

func (s3fs *S3FS) Walk(path string, vistorFunction FileVisitFunction) error

Walk will traverse an s3 file system recursively, starting at the provided prefix, and apply the visitorFunction to each s3 object

func (*S3FS) WriteChunk

func (s3fs *S3FS) WriteChunk(u UploadConfig) (UploadResult, error)

type S3FSConfig

type S3FSConfig struct {
	S3Id     string
	S3Key    string
	S3Region string
	S3Bucket string
}

S3FSConfig stores the configuration and credentials necessary to create an s3 instance of the filestore

type S3FileInfo

type S3FileInfo struct {
	// contains filtered or unexported fields
}

S3FileInfo is a wrapper around the s3.Object struct that implements the os.FileInfo interface

func (*S3FileInfo) IsDir

func (obj *S3FileInfo) IsDir() bool

IsDir returns a boolean determining whether an object is a directory or not

func (*S3FileInfo) ModTime

func (obj *S3FileInfo) ModTime() time.Time

ModTime returns the time the s3 object was last modified

func (*S3FileInfo) Mode

func (obj *S3FileInfo) Mode() os.FileMode

Mode defaults to Irregular

func (*S3FileInfo) Name

func (obj *S3FileInfo) Name() string

Name returns the file name of the s3 object

func (*S3FileInfo) Size

func (obj *S3FileInfo) Size() int64

Size returns the file size in bytes

func (*S3FileInfo) Sys

func (obj *S3FileInfo) Sys() interface{}

Sys defaults to nil for objects of s3

type UploadConfig

type UploadConfig struct {
	//PathInfo   models.ModelPathInfo
	//DirPath    string
	//FilePath   string
	ObjectPath string
	//ObjectName string
	ChunkId int64
	//FileId     uuid.UUID
	UploadId string
	Data     []byte
}

type UploadResult

type UploadResult struct {
	ID         string `json:"id"`
	WriteSize  int    `json:"size"`
	IsComplete bool   `json:"isComplete"`
}