storage

package
v0.37.0 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Apr 22, 2024 License: Apache-2.0 Imports: 19 Imported by: 36

README

Storage API

Deprecated - please use https://github.com/viant/afs API instead

This API provides unified way of accessing any storage system.

It comes with the following implementation so far:

<a name="import>

URL Scheme Description Pacakge
file Local file system github.com/viant/toolbox/storage
https HTTP/s based system github.com/viant/toolbox/storage
mem Memory file system github.com/viant/toolbox/storage
scp SCP/SSH base systm github.com/viant/toolbox/storage/scp
s3 Amazon Web Service S3 github.com/viant/toolbox/storage/aws
gs Google Storage github.com/viant/toolbox/storage/gs

type Service interface {
	//List returns a list of object for supplied url
	List(URL string) ([]Object, error)

	//Exists returns true if resource exists
	Exists(URL string) (bool, error)

	//Object returns a Object for supplied url
	StorageObject(URL string) (Object, error)

	//Download returns reader for downloaded storage object
	Download(object Object) (io.ReadCloser, error)

	//Upload uploads provided reader content for supplied storage object.
	Upload(URL string, reader io.Reader) error

	//Delete removes passed in storage object
	Delete(object Object) error

	//Register register schema with provided service
	Register(schema string, service Service) error

	//Closes storage service
	Close() error
}


//Object represents a storage object
type Object interface {
	//URL return storage url
	URL() string

	//Type returns storage type  StorageObjectFolderType or StorageObjectContentType
	Type() int

	//IsFolder returns true if object is a folder
	IsFolder() bool

	//IsContent returns true if object is a file
	IsContent() bool

	//Wrap wraps source storage object
	Wrap(source interface{})

	//Unwrap unwraps source storage object into provided target.
	Unwrap(target interface{}) error

	FileInfo() os.FileInfo
}

Usage:

    import (
    	"github.com/viant/toolbox/storage"
    	_ "github.com/viant/toolbox/storage/gs"
    	_ "github.com/viant/toolbox/storage/s3"
	
    )

    destinationURL := "gs://myBucket/set1/content.gz"
    destinationCredentialFile = "gs-secret.json"
	storageService, err := storage.NewServiceForURL(destinationURL, destinationCredentialFile)

    provider := storage.Registry().Get("s3")
    storageS3Service, err := provider("aws-secret.json")

Documentation

Overview

Package storage define abstract storage operation Deprecated - please use https://github.com/viant/afs API instead This package is frozen and no new functionality will be added, and future removal takes place.

Index

Constants

View Source
const (
	StorageObjectFolderType  int //folder type
	StorageObjectContentType     //file type
)
View Source
const FileProviderSchema = "file"
View Source
const HttpProviderScheme = "http"
View Source
const HttpsProviderScheme = "https"
View Source
const MemoryProviderScheme = "mem"

Variables

View Source
var DefaultFileMode os.FileMode = 0755
View Source
var HTTPClientProvider = func() (*http.Client, error) {
	return toolbox.NewHttpClient(&toolbox.HttpOptions{Key: "MaxIdleConns", Value: 0})
}

HTTPClientProvider represents http client provider

View Source
var MemoryRoot = newMemoryFolder("mem:///", NewFileInfo("/", 102, folderMode, time.Now(), true))

MemoryRoot represents memory root storage

Functions

func Archive

func Archive(service Service, URL string, writer *zip.Writer) error

Archive archives supplied URL assets into zip writer

func ArchiveWithFilter added in v0.15.0

func ArchiveWithFilter(service Service, URL string, writer *zip.Writer, predicate func(candidate Object) bool) error

Archive archives supplied URL assets into zip writer with supplied filter

func Copy

func Copy(sourceService Service, sourceURL string, destinationService Service, destinationURL string, modifyContentHandler ModificationHandler, copyHandler CopyHandler) (err error)

Copy downloads objects from source URL to upload them to destination URL.

func Download

func Download(service Service, URL string) (io.ReadCloser, error)

Download returns a download reader for supplied URL

func DownloadText

func DownloadText(service Service, URL string) (string, error)

DownloadText returns a text for supplied URL

func GenerateStorageCode

func GenerateStorageCode(mappings ...*StorageMapping) error

GenerateStorageCode create a *.go files with statically scanned content from source URL.

func NewFileInfo

func NewFileInfo(name string, size int64, mode os.FileMode, modificationTime time.Time, isDir bool) os.FileInfo

func NewFileMode

func NewFileMode(fileAttributes string) (os.FileMode, error)

func Parse added in v0.35.0

func Parse(URL string) (*url.URL, error)

func Registry added in v0.23.0

func Registry() *registry

Registry returns new provider

func ResetMemory

func ResetMemory()

ResetMemory reset memory root storage

func Tar added in v0.19.0

func Tar(service Service, URL string, writer *tar.Writer, includeOwnerDir bool) error

Tar tar archives supplied URL assets into zip writer

Types

type AbstractObject

type AbstractObject struct {
	Object

	Source interface{}
	// contains filtered or unexported fields
}

AbstractObject represents abstract storage object

func NewAbstractStorageObject

func NewAbstractStorageObject(url string, source interface{}, fileInfo os.FileInfo) *AbstractObject

NewAbstractStorageObject creates a new abstract storage object

func (*AbstractObject) FileInfo

func (o *AbstractObject) FileInfo() os.FileInfo

func (*AbstractObject) IsContent

func (o *AbstractObject) IsContent() bool

IsContent returns true if object is a file

func (*AbstractObject) IsFolder

func (o *AbstractObject) IsFolder() bool

IsFolder returns true if object is a folder

func (*AbstractObject) Type

func (o *AbstractObject) Type() int

Type returns storage type StorageObjectFolderType or StorageObjectContentType

func (*AbstractObject) URL

func (o *AbstractObject) URL() string

URL return storage url

func (*AbstractObject) Wrap

func (o *AbstractObject) Wrap(source interface{})

Wrap wraps source storage object

type CopyHandler

type CopyHandler func(sourceObject Object, source io.Reader, destinationService Service, destinationURL string) error

func NewStorageMapperHandler

func NewStorageMapperHandler(filename, pkg string, useTextFormat bool, binaryFormat map[string]bool) (CopyHandler, io.WriteCloser, error)

NewStorageMapperHandler creates a template handler for generating go file that write static content into memory service.

type MemoryFile

type MemoryFile struct {
	// contains filtered or unexported fields
}

func (*MemoryFile) Object

func (f *MemoryFile) Object() Object

type MemoryFolder

type MemoryFolder struct {
	// contains filtered or unexported fields
}

func (*MemoryFolder) Object

func (f *MemoryFolder) Object() Object

func (*MemoryFolder) Objects

func (f *MemoryFolder) Objects() []Object

type ModificationHandler

type ModificationHandler func(reader io.ReadCloser) (io.ReadCloser, error)

type Object

type Object interface {
	//URL return storage url
	URL() string
	//Type returns storage type either folder or file
	Type() int
	//IsFolder returns true if object is a folder
	IsFolder() bool
	//IsContent returns true if object is a file
	IsContent() bool
	//Wrap wraps source storage object
	Wrap(source interface{})
	//Unwrap unwraps source storage object into provided target.
	Unwrap(target interface{}) error
	FileInfo() os.FileInfo
}

Object represents a storage object

type Provider

type Provider func(credentialFile string) (Service, error)

Provider represetns a service provider

type Service

type Service interface {
	//List returns a list of object for supplied url
	List(URL string) ([]Object, error)
	//Exists returns true if resource exists
	Exists(URL string) (bool, error)
	//Object returns a Object for supplied url
	StorageObject(URL string) (Object, error)
	//Download returns reader for downloaded storage object
	Download(object Object) (io.ReadCloser, error)
	//DownloadWithURL returns reader for downloaded URL object
	DownloadWithURL(URL string) (io.ReadCloser, error)
	//Upload uploads provided reader content for supplied storage object.
	Upload(URL string, reader io.Reader) error
	//Upload uploads provided reader content for supplied storage object.
	UploadWithMode(URL string, mode os.FileMode, reader io.Reader) error
	//Delete removes passed in storage object
	Delete(object Object) error
	//Register register schema with provided service
	Register(schema string, service Service) error
	//Closes storage service
	Close() error
}

Service represents abstract way to accessing local or remote storage

func NewFileStorage

func NewFileStorage() Service

func NewHttpStorageService

func NewHttpStorageService(credential *cred.Config) Service

func NewMemoryService

func NewMemoryService() Service

creates a new memory service

func NewPrivateMemoryService

func NewPrivateMemoryService() Service

creates a new private memory service

func NewService

func NewService() Service

NewService creates a new storage service

func NewServiceForURL

func NewServiceForURL(URL, credentials string) (Service, error)

NewServiceForURL creates a new storage service for provided URL scheme and optional credential file

type StorageMapping

type StorageMapping struct {
	SourceURL        string
	SourceCredential string
	DestinationURI   string
	TargetFile       string
	TargetPackage    string
	UseTextFormat    bool
}

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL