storage

README

Storage API

This API provides unified way of accessing any storage system.

It comes with the following implementation so far:

1. Local file system
2. SCP/SSH
3. Memory file system
4. Google Storage
5. Amazon Web Service S3
6. HTTP/S

type Service interface {
	//List returns a list of object for supplied url
	List(URL string) ([]Object, error)

	//Exists returns true if resource exists
	Exists(URL string) (bool, error)

	//Object returns a Object for supplied url
	StorageObject(URL string) (Object, error)

	//Download returns reader for downloaded storage object
	Download(object Object) (io.ReadCloser, error)

	//Upload uploads provided reader content for supplied storage object.
	Upload(URL string, reader io.Reader) error

	//Delete removes passed in storage object
	Delete(object Object) error

	//Register register schema with provided service
	Register(schema string, service Service) error

	//Closes storage service
	Close() error
}


//Object represents a storage object
type Object interface {
	//URL return storage url
	URL() string

	//Type returns storage type  StorageObjectFolderType or StorageObjectContentType
	Type() int

	//IsFolder returns true if object is a folder
	IsFolder() bool

	//IsContent returns true if object is a file
	IsContent() bool

	//Wrap wraps source storage object
	Wrap(source interface{})

	//Unwrap unwraps source storage object into provided target.
	Unwrap(target interface{}) error

	FileInfo() os.FileInfo
}

Usage

    import (
    	"github.com/viant/toolbox/storage"
    	_ "github.com/viant/toolbox/storage/gs"
	
    )

    destinationURL := "gs://myBucket/set1/content.gz"
    destinationCredentialFile = "gs-secret.json"
	storageService, err := storage.NewServiceForURL(destinationURL, destinationCredentialFile)

Documentation

Index

• Constants
• Variables
• func Copy(sourceService Service, sourceURL string, destinationService Service, ...) (err error)
• func GenerateStorageCode(mappings ...*StorageMapping) error
• func NewFileInfo(name string, size int64, mode os.FileMode, modificationTime time.Time, ...) os.FileInfo
• func NewFileMode(fileAttributes string) (os.FileMode, error)
• type AbstractObject
  • func NewAbstractStorageObject(url string, source interface{}, fileInfo os.FileInfo) *AbstractObject
  • func (o *AbstractObject) FileInfo() os.FileInfo
  • func (o *AbstractObject) IsContent() bool
  • func (o *AbstractObject) IsFolder() bool
  • func (o *AbstractObject) Type() int
  • func (o *AbstractObject) URL() string
  • func (o *AbstractObject) Wrap(source interface{})
• type CopyHandler
  • func NewStorageMapperHandler(filename, pkg string) (CopyHandler, io.WriteCloser, error)
• type MemoryFile
  • func (f *MemoryFile) Object() Object
• type MemoryFolder
  • func (f *MemoryFolder) Object() Object
  • func (f *MemoryFolder) Objects() []Object
• type ModificationHandler
• type Object
• type Provider
• type Service
  • func NewFileStorage() Service
  • func NewHttpStorageService(credential *cred.Config) Service
  • func NewMemoryService() Service
  • func NewService() Service
  • func NewServiceForURL(URL, credentialFile string) (Service, error)
• type StorageMapping
• type StorageProvider
  • func NewStorageProvider() *StorageProvider
  • func (p *StorageProvider) Get(namespace string) func(credentialFile string) (Service, error)

Constants

const (
	StorageObjectFolderType  int //folder type
	StorageObjectContentType     //file type
)

const HttpProviderScheme = "http"

const HttpsProviderScheme = "https"

const MemoryProviderScheme = "mem"

Variables

var MemoryRoot = newMemoryFolder("mem:///", NewFileInfo("/", 102, folderMode, time.Now(), true))

MemoryRoot represents memory root storage

Functions

func Copy

func Copy(sourceService Service, sourceURL string, destinationService Service, destinationURL string, modifyContentHandler ModificationHandler, copyHandler CopyHandler) (err error)

Copy downloads objects from source URL to upload them to destination URL.

func GenerateStorageCode

func GenerateStorageCode(mappings ...*StorageMapping) error

GenerateStorageCode create a *.go files with statically scanned content from source URL.

func NewFileInfo

func NewFileInfo(name string, size int64, mode os.FileMode, modificationTime time.Time, isDir bool) os.FileInfo

func NewFileMode

func NewFileMode(fileAttributes string) (os.FileMode, error)

Types

type AbstractObject

type AbstractObject struct {
	Object

	Source interface{}
	// contains filtered or unexported fields
}

AbstractObject represents abstract storage object

func NewAbstractStorageObject

func NewAbstractStorageObject(url string, source interface{}, fileInfo os.FileInfo) *AbstractObject

NewAbstractStorageObject creates a new abstract storage object

func (*AbstractObject) FileInfo

func (o *AbstractObject) FileInfo() os.FileInfo

func (*AbstractObject) IsContent

func (o *AbstractObject) IsContent() bool

IsContent returns true if object is a file

func (*AbstractObject) IsFolder

func (o *AbstractObject) IsFolder() bool

IsFolder returns true if object is a folder

func (*AbstractObject) Type

func (o *AbstractObject) Type() int

Type returns storage type StorageObjectFolderType or StorageObjectContentType

func (*AbstractObject) URL

func (o *AbstractObject) URL() string

URL return storage url

func (*AbstractObject) Wrap

func (o *AbstractObject) Wrap(source interface{})

Wrap wraps source storage object

type CopyHandler

type CopyHandler func(sourceObject Object, source io.Reader, destinationService Service, destinationURL string) error

func NewStorageMapperHandler

func NewStorageMapperHandler(filename, pkg string) (CopyHandler, io.WriteCloser, error)

NewStorageMapperHandler creates a template handler for generating go file that write static content into memory service.

type MemoryFile

type MemoryFile struct {
	// contains filtered or unexported fields
}

func (*MemoryFile) Object

func (f *MemoryFile) Object() Object

type MemoryFolder

type MemoryFolder struct {
	// contains filtered or unexported fields
}

func (*MemoryFolder) Object

func (f *MemoryFolder) Object() Object

func (*MemoryFolder) Objects

func (f *MemoryFolder) Objects() []Object

type ModificationHandler

type ModificationHandler func(reader io.ReadCloser) (io.ReadCloser, error)

type Object

type Object interface {
	//URL return storage url
	URL() string

	//Type returns storage type  StorageObjectFolderType or StorageObjectContentType
	Type() int

	//IsFolder returns true if object is a folder
	IsFolder() bool

	//IsContent returns true if object is a file
	IsContent() bool

	//Wrap wraps source storage object
	Wrap(source interface{})

	//Unwrap unwraps source storage object into provided target.
	Unwrap(target interface{}) error

	FileInfo() os.FileInfo
}

Object represents a storage object

type Provider

type Provider func(credentialFile string) (Service, error)

type Service

type Service interface {
	//List returns a list of object for supplied url
	List(URL string) ([]Object, error)

	//Exists returns true if resource exists
	Exists(URL string) (bool, error)

	//Object returns a Object for supplied url
	StorageObject(URL string) (Object, error)

	//Download returns reader for downloaded storage object
	Download(object Object) (io.ReadCloser, error)

	//Upload uploads provided reader content for supplied storage object.
	Upload(URL string, reader io.Reader) error

	//Delete removes passed in storage object
	Delete(object Object) error

	//Register register schema with provided service
	Register(schema string, service Service) error

	//Closes storage service
	Close() error
}

Service represents abstract way to accessing local or remote storage

func NewFileStorage

func NewFileStorage() Service

func NewHttpStorageService

func NewHttpStorageService(credential *cred.Config) Service

func NewMemoryService

func NewMemoryService() Service

func NewService

func NewService() Service

NewService creates a new storage service

func NewServiceForURL

func NewServiceForURL(URL, credentialFile string) (Service, error)

NewServiceForURL creates a new storage service for provided URL scheme and optional credential file

type StorageMapping

type StorageMapping struct {
	SourceURL        string
	SourceCredential string
	DestinationURI   string
	TargetFile       string
	TargetPackage    string
}

type StorageProvider

type StorageProvider struct {
	Registry map[string]Provider
}

func NewStorageProvider

func NewStorageProvider() *StorageProvider

func (*StorageProvider) Get

func (p *StorageProvider) Get(namespace string) func(credentialFile string) (Service, error)