filesys

package
v0.3.81 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Aug 19, 2022 License: GPL-3.0 Imports: 39 Imported by: 2

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

func GetDataflow 

func GetDataflow(fs FileSysClient, paths []string, cfg FileStreamConfig) (df *iop.Dataflow, err error)

GetDataflow returns a dataflow from specified paths in specified FileSysClient

func MakeDatastream 

func MakeDatastream(reader io.Reader) (ds *iop.Datastream, err error)

MakeDatastream create a datastream from a reader

func ParseURL 

func ParseURL(urlStr string) (host string, path string, err error)

ParseURL parses a URL

func TestFsPermissions 

func TestFsPermissions(fs FileSysClient, pathURL string) (err error)

TestFsPermissions tests read/write permisions

func Write 

func Write(reader io.Reader, writer io.Writer) (bw int64, err error)

Write writer to a writer from a reader

func WriteDatastream 

func WriteDatastream(writer io.Writer, ds *iop.Datastream) (bw int64, err error)

WriteDatastream writes a datasream to a writer or use fs.Write(path, ds.NewCsvReader(0))

Types

type AzureFileSysClient 

type AzureFileSysClient struct {
	BaseFileSysClient
	// contains filtered or unexported fields
}

AzureFileSysClient is a file system client to write file to Microsoft's Azure file sys.

func (*AzureFileSysClient) Buckets added in v0.3.12 

func (fs *AzureFileSysClient) Buckets() (paths []string, err error)

Buckets returns the containers found in the project

func (*AzureFileSysClient) Connect 

func (fs *AzureFileSysClient) Connect() (err error)

Connect initiates the fs client connection

func (*AzureFileSysClient) Delete 

func (fs *AzureFileSysClient) Delete(urlStr string) (err error)

Delete list objects in path

func (*AzureFileSysClient) GetReader 

func (fs *AzureFileSysClient) GetReader(urlStr string) (reader io.Reader, err error)

GetReader returns an Azure FS reader

func (*AzureFileSysClient) Init 

func (fs *AzureFileSysClient) Init(ctx context.Context) (err error)

Init initializes the fs client

func (*AzureFileSysClient) List 

func (fs *AzureFileSysClient) List(url string) (paths []string, err error)

List list objects in path

func (*AzureFileSysClient) ListRecursive 

func (fs *AzureFileSysClient) ListRecursive(url string) (paths []string, err error)

ListRecursive list objects in path

func (*AzureFileSysClient) Write 

func (fs *AzureFileSysClient) Write(urlStr string, reader io.Reader) (bw int64, err error)

type BaseFileSysClient 

type BaseFileSysClient struct {
	FileSysClient
	// contains filtered or unexported fields
}

BaseFileSysClient is the base file system type.

func (*BaseFileSysClient) Buckets added in v0.3.12 

func (fs *BaseFileSysClient) Buckets() (paths []string, err error)

Buckets returns the buckets found in the account

func (*BaseFileSysClient) Client 

func (fs *BaseFileSysClient) Client() *BaseFileSysClient

Client provides a pointer to itself

func (*BaseFileSysClient) Context 

func (fs *BaseFileSysClient) Context() (context *g.Context)

Context provides a pointer to context

func (*BaseFileSysClient) FsType 

func (fs *BaseFileSysClient) FsType() dbio.Type

FsType return the type of the client

func (*BaseFileSysClient) GetDatastream 

func (fs *BaseFileSysClient) GetDatastream(urlStr string) (ds *iop.Datastream, err error)

GetDatastream return a datastream for the given path

func (*BaseFileSysClient) GetProp 

func (fs *BaseFileSysClient) GetProp(key string) string

GetProp returns the value of a property

func (*BaseFileSysClient) GetReaders 

func (fs *BaseFileSysClient) GetReaders(paths ...string) (readers []io.Reader, err error)

GetReaders returns one or more readers from specified paths in specified FileSysClient

func (*BaseFileSysClient) ReadDataflow 

func (fs *BaseFileSysClient) ReadDataflow(url string, cfg ...FileStreamConfig) (df *iop.Dataflow, err error)

ReadDataflow read

func (*BaseFileSysClient) Self 

func (fs *BaseFileSysClient) Self() FileSysClient

Instance returns the respective connection Instance This is useful to refer back to a subclass method from the superclass level. (Aka overloading)

func (*BaseFileSysClient) SetProp 

func (fs *BaseFileSysClient) SetProp(key string, val string)

SetProp sets the value of a property

func (*BaseFileSysClient) WriteDataflow 

func (fs *BaseFileSysClient) WriteDataflow(df *iop.Dataflow, url string) (bw int64, err error)

WriteDataflow writes a dataflow to a file sys.

func (*BaseFileSysClient) WriteDataflowReady 

func (fs *BaseFileSysClient) WriteDataflowReady(df *iop.Dataflow, url string, fileReadyChn chan string) (bw int64, err error)

WriteDataflowReady writes to a file sys and notifies the fileReady chan.

type Excel 

type Excel struct {
	File   *excelize.File
	Sheets []string
	Path   string
	// contains filtered or unexported fields
}

Excel represent an Excel object pointing to its file

func NewExcel 

func NewExcel() (xls *Excel)

NewExcel creates a new excel file

func NewExcelFromFile 

func NewExcelFromFile(path string) (xls *Excel, err error)

NewExcelFromFile return a new Excel instance from a local file

func NewExcelFromReader 

func NewExcelFromReader(reader io.Reader) (xls *Excel, err error)

NewExcelFromReader return a new Excel instance from a reader

func (*Excel) GetDataset 

func (xls *Excel) GetDataset(sheet string) (data iop.Dataset)

GetDataset returns a dataset of the provided sheet

func (*Excel) GetDatasetFromRange 

func (xls *Excel) GetDatasetFromRange(sheet, cellRange string) (data iop.Dataset, err error)

GetDatasetFromRange returns a dataset of the provided sheet / range cellRange example: `$AH$13:$AI$20` or `AH13:AI20` or `A:E`

func (*Excel) RefreshSheets 

func (xls *Excel) RefreshSheets() (err error)

RefreshSheets refresh sheet index data

func (*Excel) WriteSheet 

func (xls *Excel) WriteSheet(shtName string, ds *iop.Datastream, mode string) (err error)

WriteSheet write a datastream into a sheet mode can be: `new`, `append` or `overwrite`. Default is `new`

func (*Excel) WriteToFile 

func (xls *Excel) WriteToFile(path string) (err error)

WriteToFile write to a file

func (*Excel) WriteToWriter 

func (xls *Excel) WriteToWriter(w io.Writer) (err error)

WriteToWriter write to a provided writer

type FileStreamConfig added in v0.3.49 

type FileStreamConfig struct {
	Limit   int
	Columns []string
}

type FileSysClient 

type FileSysClient interface {
	Self() FileSysClient
	Init(ctx context.Context) (err error)
	Client() *BaseFileSysClient
	Context() (context *g.Context)
	FsType() dbio.Type
	Delete(path string) (err error)
	GetReader(path string) (reader io.Reader, err error)
	GetReaders(paths ...string) (readers []io.Reader, err error)
	GetDatastream(path string) (ds *iop.Datastream, err error)
	GetWriter(path string) (writer io.Writer, err error)
	Buckets() (paths []string, err error)
	List(path string) (paths []string, err error)
	ListRecursive(path string) (paths []string, err error)
	Write(path string, reader io.Reader) (bw int64, err error)

	ReadDataflow(url string, cfg ...FileStreamConfig) (df *iop.Dataflow, err error)
	WriteDataflow(df *iop.Dataflow, url string) (bw int64, err error)
	WriteDataflowReady(df *iop.Dataflow, url string, fileReadyChn chan string) (bw int64, err error)
	GetProp(key string) (val string)
	SetProp(key string, val string)
	MkdirAll(path string) (err error)
}

FileSysClient is a client to a file systems such as local, s3, hdfs, azure storage, google cloud storage

func NewFileSysClient 

func NewFileSysClient(fst dbio.Type, props ...string) (fsClient FileSysClient, err error)

NewFileSysClient create a file system client such as local, s3, azure storage, google cloud storage props are provided as `"Prop1=Value1", "Prop2=Value2", ...`

func NewFileSysClientContext 

func NewFileSysClientContext(ctx context.Context, fst dbio.Type, props ...string) (fsClient FileSysClient, err error)

NewFileSysClientContext create a file system client with context such as local, s3, azure storage, google cloud storage props are provided as `"Prop1=Value1", "Prop2=Value2", ...`

func NewFileSysClientFromURL 

func NewFileSysClientFromURL(url string, props ...string) (fsClient FileSysClient, err error)

NewFileSysClientFromURL returns the proper fs client for the given path props are provided as `"Prop1=Value1", "Prop2=Value2", ...`

func NewFileSysClientFromURLContext 

func NewFileSysClientFromURLContext(ctx context.Context, url string, props ...string) (fsClient FileSysClient, err error)

NewFileSysClientFromURLContext returns the proper fs client for the given path with context props are provided as `"Prop1=Value1", "Prop2=Value2", ...`

type GoogleFileSysClient 

type GoogleFileSysClient struct {
	BaseFileSysClient
	// contains filtered or unexported fields
}

GoogleFileSysClient is a file system client to write file to Amazon's S3 file sys.

func (*GoogleFileSysClient) Buckets added in v0.3.12 

func (fs *GoogleFileSysClient) Buckets() (paths []string, err error)

Buckets returns the buckets found in the project

func (*GoogleFileSysClient) Connect 

func (fs *GoogleFileSysClient) Connect() (err error)

Connect initiates the Google Cloud Storage client

func (*GoogleFileSysClient) Delete 

func (fs *GoogleFileSysClient) Delete(urlStr string) (err error)

Delete list objects in path

func (*GoogleFileSysClient) GetReader 

func (fs *GoogleFileSysClient) GetReader(path string) (reader io.Reader, err error)

GetReader returns the reader for the given path

func (*GoogleFileSysClient) Init 

func (fs *GoogleFileSysClient) Init(ctx context.Context) (err error)

Init initializes the fs client

func (*GoogleFileSysClient) List 

func (fs *GoogleFileSysClient) List(path string) (paths []string, err error)

List returns the list of objects

func (*GoogleFileSysClient) ListRecursive 

func (fs *GoogleFileSysClient) ListRecursive(path string) (paths []string, err error)

ListRecursive returns the list of objects recursively

func (*GoogleFileSysClient) Write 

func (fs *GoogleFileSysClient) Write(path string, reader io.Reader) (bw int64, err error)

type GoogleSheet 

type GoogleSheet struct {
	Sheets        []string
	SpreadsheetID string
	// contains filtered or unexported fields
}

GoogleSheet represent a Google Sheet object

func NewGoogleSheet 

func NewGoogleSheet(props ...string) (ggs *GoogleSheet, err error)

NewGoogleSheet is a blank spreadsheet title is the new spreadsheet title

func NewGoogleSheetFromURL 

func NewGoogleSheetFromURL(urlStr string, props ...string) (ggs *GoogleSheet, err error)

NewGoogleSheetFromURL return a new GoogleSheet instance from a provided url

func (*GoogleSheet) GetDataset 

func (ggs *GoogleSheet) GetDataset(shtName string) (data iop.Dataset, err error)

GetDataset returns a dataset of the sheet

func (*GoogleSheet) GetDatasetFromRange 

func (ggs *GoogleSheet) GetDatasetFromRange(shtName, cellRange string) (data iop.Dataset, err error)

GetDatasetFromRange returns a dataset from the specified range

func (*GoogleSheet) RefreshSheets 

func (ggs *GoogleSheet) RefreshSheets() (err error)

RefreshSheets refreshes sheets data

func (*GoogleSheet) URL 

func (ggs *GoogleSheet) URL() string

func (*GoogleSheet) WriteSheet 

func (ggs *GoogleSheet) WriteSheet(shtName string, ds *iop.Datastream, mode string) (err error)

WriteSheet write a datastream into a sheet mode can be: `new`, `append` or `overwrite`. Default is `new`

type HTTPFileSysClient 

type HTTPFileSysClient struct {
	BaseFileSysClient
	// contains filtered or unexported fields
}

HTTPFileSysClient is for HTTP files

func (*HTTPFileSysClient) Connect 

func (fs *HTTPFileSysClient) Connect() (err error)

Connect initiates the http client

func (*HTTPFileSysClient) Delete 

func (fs *HTTPFileSysClient) Delete(path string) (err error)

Delete :

func (*HTTPFileSysClient) GetReader 

func (fs *HTTPFileSysClient) GetReader(url string) (reader io.Reader, err error)

GetReader gets a reader for an HTTP resource (download)

func (*HTTPFileSysClient) Init 

func (fs *HTTPFileSysClient) Init(ctx context.Context) (err error)

Init initializes the fs client

func (*HTTPFileSysClient) List 

func (fs *HTTPFileSysClient) List(url string) (paths []string, err error)

List lists all urls on the page

func (*HTTPFileSysClient) ListRecursive 

func (fs *HTTPFileSysClient) ListRecursive(url string) (paths []string, err error)

ListRecursive lists all urls on the page

func (*HTTPFileSysClient) Write 

func (fs *HTTPFileSysClient) Write(urlStr string, reader io.Reader) (bw int64, err error)

Write uploads an HTTP file

type LocalFileSysClient 

type LocalFileSysClient struct {
	BaseFileSysClient
	// contains filtered or unexported fields
}

LocalFileSysClient is a file system client to write file to local file sys.

func (*LocalFileSysClient) Delete 

func (fs *LocalFileSysClient) Delete(path string) (err error)

Delete deletes the given path (file or directory)

func (*LocalFileSysClient) GetDatastream 

func (fs *LocalFileSysClient) GetDatastream(path string) (ds *iop.Datastream, err error)

GetDatastream return a datastream for the given path

func (*LocalFileSysClient) GetReader 

func (fs *LocalFileSysClient) GetReader(path string) (reader io.Reader, err error)

GetReader return a reader for the given path

func (*LocalFileSysClient) GetWriter 

func (fs *LocalFileSysClient) GetWriter(path string) (writer io.Writer, err error)

GetWriter creates the file if non-existent and return a writer

func (*LocalFileSysClient) Init 

func (fs *LocalFileSysClient) Init(ctx context.Context) (err error)

Init initializes the fs client

func (*LocalFileSysClient) List 

func (fs *LocalFileSysClient) List(path string) (paths []string, err error)

List lists the file in given directory path

func (*LocalFileSysClient) ListRecursive 

func (fs *LocalFileSysClient) ListRecursive(path string) (paths []string, err error)

ListRecursive lists the file in given directory path recursively

func (*LocalFileSysClient) MkdirAll 

func (fs *LocalFileSysClient) MkdirAll(path string) (err error)

MkdirAll creates child directories

func (*LocalFileSysClient) Write 

func (fs *LocalFileSysClient) Write(path string, reader io.Reader) (bw int64, err error)

Write creates the file if non-existent and writes from the reader

type PathNode 

type PathNode struct {
	Name         string    `json:"name"`
	IsDir        bool      `json:"is_dir"`
	Size         int64     `json:"size,omitempty"`
	LastModified time.Time `json:"last_modified,omitempty"`
	Children     PathNodes `json:"children,omitempty"`
}

PathNode represents a file node

type PathNodes 

type PathNodes []PathNode

PathNodes represent file nodes

func (*PathNodes) Add 

func (pn *PathNodes) Add(p PathNode)

Add adds a new node to list

func (PathNodes) List 

func (pn PathNodes) List() (paths []string)

List give a list of recursive paths

type S3FileSysClient 

type S3FileSysClient struct {
	BaseFileSysClient

	RegionMap map[string]string
	// contains filtered or unexported fields
}

S3FileSysClient is a file system client to write file to Amazon's S3 file sys.

func (*S3FileSysClient) Buckets added in v0.3.12 

func (fs *S3FileSysClient) Buckets() (paths []string, err error)

Buckets returns the buckets found in the account

func (*S3FileSysClient) Connect 

func (fs *S3FileSysClient) Connect() (err error)

Connect initiates the Google Cloud Storage client

func (*S3FileSysClient) Delete 

func (fs *S3FileSysClient) Delete(path string) (err error)

Delete deletes the given path (file or directory) path should specify the full path with scheme: `s3://my_bucket/key/to/file.txt`

func (*S3FileSysClient) GetReader 

func (fs *S3FileSysClient) GetReader(path string) (reader io.Reader, err error)

GetReader return a reader for the given path path should specify the full path with scheme: `s3://my_bucket/key/to/file.txt` or `s3://my_bucket/key/to/directory`

func (*S3FileSysClient) GetWriter 

func (fs *S3FileSysClient) GetWriter(path string) (writer io.Writer, err error)

GetWriter creates the file if non-existent and return a writer path should specify the full path with scheme: `s3://my_bucket/key/to/file.txt`

func (*S3FileSysClient) Init 

func (fs *S3FileSysClient) Init(ctx context.Context) (err error)

Init initializes the fs client

func (*S3FileSysClient) List 

func (fs *S3FileSysClient) List(path string) (paths []string, err error)

List lists the file in given directory path path should specify the full path with scheme: `s3://my_bucket/key/to/directory`

func (*S3FileSysClient) ListRecursive 

func (fs *S3FileSysClient) ListRecursive(path string) (paths []string, err error)

ListRecursive lists the file in given directory path recusively path should specify the full path with scheme: `s3://my_bucket/key/to/directory`

func (*S3FileSysClient) Write 

func (fs *S3FileSysClient) Write(path string, reader io.Reader) (bw int64, err error)

type SftpFileSysClient 

type SftpFileSysClient struct {
	BaseFileSysClient
	// contains filtered or unexported fields
}

SftpFileSysClient is for SFTP / SSH file ops

func (*SftpFileSysClient) Connect 

func (fs *SftpFileSysClient) Connect() (err error)

Connect initiates the Google Cloud Storage client

func (*SftpFileSysClient) Delete 

func (fs *SftpFileSysClient) Delete(urlStr string) (err error)

Delete list objects in path

func (*SftpFileSysClient) GetReader 

func (fs *SftpFileSysClient) GetReader(urlStr string) (reader io.Reader, err error)

GetReader return a reader for the given path

func (*SftpFileSysClient) GetWriter 

func (fs *SftpFileSysClient) GetWriter(urlStr string) (writer io.Writer, err error)

GetWriter creates the file if non-existent and return a writer

func (*SftpFileSysClient) Init 

func (fs *SftpFileSysClient) Init(ctx context.Context) (err error)

Init initializes the fs client

func (*SftpFileSysClient) List 

func (fs *SftpFileSysClient) List(url string) (paths []string, err error)

List list objects in path

func (*SftpFileSysClient) ListRecursive 

func (fs *SftpFileSysClient) ListRecursive(url string) (paths []string, err error)

ListRecursive list objects in path recursively

func (*SftpFileSysClient) MkdirAll 

func (fs *SftpFileSysClient) MkdirAll(path string) (err error)

MkdirAll creates child directories

func (*SftpFileSysClient) Write 

func (fs *SftpFileSysClient) Write(urlStr string, reader io.Reader) (bw int64, err error)

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.