storage

Documentation

Index

• Constants
• Variables
• func AsSha256(o interface{}) string
• func Create(fileName string) (*os.File, error)
• func FileExists(filename string) bool
• func GetAllProtocol() (protocols []string)
• func RemoveDir(dir string) error
• type GCSObjectDownloader
  • func (g *GCSObjectDownloader) Download(client stiface.Client, it stiface.ObjectIterator) error
  • func (g *GCSObjectDownloader) DownloadFile(client stiface.Client, attrs *gstorage.ObjectAttrs, file *os.File) error
  • func (g *GCSObjectDownloader) GetObjectIterator(client stiface.Client) (stiface.ObjectIterator, error)
  • func (g *GCSObjectDownloader) WriteToFile(data []byte, attrs *gstorage.ObjectAttrs, file *os.File) error
• type GCSProvider
  • func (p *GCSProvider) DownloadModel(modelDir string, modelName string, storageUri string) error
• type HTTPSDownloader
  • func (h *HTTPSDownloader) Download(client http.Client) error
• type HTTPSProvider
  • func (m *HTTPSProvider) DownloadModel(modelDir string, modelName string, storageUri string) error
• type Protocol
• type Provider
  • func GetProvider(providers map[Protocol]Provider, protocol Protocol) (Provider, error)
• type S3ObjectDownloader
  • func (s *S3ObjectDownloader) Download(objects []s3manager.BatchDownloadObject) error
  • func (s *S3ObjectDownloader) GetAllObjects(s3Svc s3iface.S3API) ([]s3manager.BatchDownloadObject, error)
• type S3Provider
  • func (m *S3Provider) DownloadModel(modelDir string, modelName string, storageUri string) error

Constants

const (
	HEADER_SUFFIX = "-headers"
)

Variables

var SupportedProtocols = []Protocol{S3, GCS, HTTPS, HTTP}

Functions

func AsSha256

func AsSha256(o interface{}) string

func Create

func Create(fileName string) (*os.File, error)

func FileExists

func FileExists(filename string) bool

func GetAllProtocol

func GetAllProtocol() (protocols []string)

func RemoveDir

func RemoveDir(dir string) error

Types

type GCSObjectDownloader

type GCSObjectDownloader struct {
	Context    context.Context
	StorageUri string
	ModelDir   string
	ModelName  string
	Bucket     string
	Item       string
}

func (*GCSObjectDownloader) Download

func (g *GCSObjectDownloader) Download(client stiface.Client, it stiface.ObjectIterator) error

func (*GCSObjectDownloader) DownloadFile

func (g *GCSObjectDownloader) DownloadFile(client stiface.Client, attrs *gstorage.ObjectAttrs, file *os.File) error

func (*GCSObjectDownloader) GetObjectIterator

func (g *GCSObjectDownloader) GetObjectIterator(client stiface.Client) (stiface.ObjectIterator, error)

func (*GCSObjectDownloader) WriteToFile

func (g *GCSObjectDownloader) WriteToFile(data []byte, attrs *gstorage.ObjectAttrs, file *os.File) error

type GCSProvider

type GCSProvider struct {
	Client stiface.Client
}

func (*GCSProvider) DownloadModel

func (p *GCSProvider) DownloadModel(modelDir string, modelName string, storageUri string) error

type HTTPSDownloader

type HTTPSDownloader struct {
	StorageUri string
	ModelDir   string
	ModelName  string
	Uri        *url.URL
}

func (*HTTPSDownloader) Download

func (h *HTTPSDownloader) Download(client http.Client) error

type HTTPSProvider

type HTTPSProvider struct {
	Client *http.Client
}

func (*HTTPSProvider) DownloadModel

func (m *HTTPSProvider) DownloadModel(modelDir string, modelName string, storageUri string) error

type Protocol

type Protocol string

const (
	S3  Protocol = "s3://"
	GCS Protocol = "gs://"
	//PVC   Protocol = "pvc://"
	//File  Protocol = "file://"
	HTTPS Protocol = "https://"
	HTTP  Protocol = "http://"
)

type Provider

type Provider interface {
	DownloadModel(modelDir string, modelName string, storageUri string) error
}

func GetProvider

func GetProvider(providers map[Protocol]Provider, protocol Protocol) (Provider, error)

type S3ObjectDownloader

type S3ObjectDownloader struct {
	StorageUri string
	ModelDir   string
	ModelName  string
	Bucket     string
	Prefix     string
	// contains filtered or unexported fields
}

func (*S3ObjectDownloader) Download

func (s *S3ObjectDownloader) Download(objects []s3manager.BatchDownloadObject) error

func (*S3ObjectDownloader) GetAllObjects

func (s *S3ObjectDownloader) GetAllObjects(s3Svc s3iface.S3API) ([]s3manager.BatchDownloadObject, error)

type S3Provider

type S3Provider struct {
	Client     s3iface.S3API
	Downloader s3manageriface.DownloadWithIterator
}

func (*S3Provider) DownloadModel

func (m *S3Provider) DownloadModel(modelDir string, modelName string, storageUri string) error