fifo

package module
v0.1.1 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Sep 28, 2019 License: MIT Imports: 19 Imported by: 0

README

Fast remote storage streaming for legacy executables

How It Works

You define some sources and/or targets as a mapping of some tag to a URL of the source/target object.

Then, using templates define a command to execute using those sources and destinations.

Internally, fifo will replace those tags with a real path on your system to a named pipes whose other end reads or writes directly to the remote object.

Examples

Backup a directory to a tar archive in S3

fifo -t archive=s3://bucket/archive.tar.gz -- tar -C /directory -cvz . -f %{archive}

Calculate the MD5 sum of a file in S3 using openssl

fifo -s input=s3://bucket/file.txt -- openssl md5 %{input}

Grep a file in S3 and upload the matches to S3

fifo -s log=s3://bucket/log-file.txt --stdout s3://bucket/grepped.txt -- grep something %{log}

Providers

file://

Opens or creates a file on the local filesystem

file://./log.txt

s3://, s3+insecure://

Downloads or uploads an object in S3.

Requires the environment parameters AWS_ACCESS_KEY, AWS_SECRET_KEY and AWS_ENDPOINT are set.

s3://bucket/path/to/file.txt

s3://bucket/path/to/file.txt?acl=public-read&type=text/plain

http://, https://

Stream a HTTP URL

https://httpbin.org/stream/1

Considerations

  • The application must read every source stream in its entirety. Seeking is not supported.

  • If an application does not open a file for reading or does not fully consume the stream then fifo may block forever (due to the nature of named pipes in UNIX).

  • Targets are destroyed automatically on failure unless --preserve is enabled.

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

func ProvideSource 

func ProvideSource(u *url.URL, providers ...Provider) (io.ReadCloser, error)

ProvideSource opens a source stream for a given URN from a given list of providers.

Types

type Call 

type Call struct {
	Executable       string
	Args             []string
	Environment      []string
	WorkingDirectory string
}

type Command 

type Command struct {
	// contains filtered or unexported fields
}

func NewCommand 

func NewCommand(t *Task) (*Command, error)

func (*Command) Start 

func (c *Command) Start(ctx context.Context) (code int, mu *MultiError)

type DestroyableFile 

type DestroyableFile struct {
	Path string
	*os.File
}

func (*DestroyableFile) Destroy 

func (f *DestroyableFile) Destroy() error

type ErrorFunc 

type ErrorFunc func() *MultiError

type FileProvider 

type FileProvider struct {
	Create os.FileMode
}

func (FileProvider) Read 

func (fp FileProvider) Read(u *url.URL) (io.ReadCloser, error)

func (FileProvider) Schema 

func (FileProvider) Schema() []string

func (FileProvider) Target 

func (FileProvider) Target(u *url.URL) string

func (FileProvider) Write 

func (fp FileProvider) Write(u *url.URL) (WriteDestroyCloser, error)

type HTTPProvider 

type HTTPProvider struct {
	Client *http.Client
}

HTTPProvider provides a source from an HTTP url

func (*HTTPProvider) Read 

func (p *HTTPProvider) Read(u *url.URL) (io.ReadCloser, error)

func (*HTTPProvider) Schema 

func (p *HTTPProvider) Schema() []string

type MultiError 

type MultiError struct {
	// contains filtered or unexported fields
}

func Catch 

func Catch(mu *MultiError, err ...error) *MultiError

func (*MultiError) Append 

func (mu *MultiError) Append(err error)

func (*MultiError) AsError 

func (mu *MultiError) AsError() error

func (*MultiError) Catch 

func (mu *MultiError) Catch(funcs ...func() error) (failure bool)

func (*MultiError) CatchMulti 

func (mu *MultiError) CatchMulti(funcs ...ErrorFunc) (failure bool)

func (*MultiError) Error 

func (mu *MultiError) Error() string

func (*MultiError) Errors 

func (mu *MultiError) Errors() []error

type NoOpWriteDestroyCloser 

type NoOpWriteDestroyCloser struct {
	io.Writer
}

func (NoOpWriteDestroyCloser) Close 

func (NoOpWriteDestroyCloser) Close() error

func (NoOpWriteDestroyCloser) Destroy 

func (NoOpWriteDestroyCloser) Destroy() error

type PipeProvider 

type PipeProvider interface {
	Source(u *url.URL) (*SourcePipe, error)
	Target(u *url.URL) (*TargetPipe, error)
}

A PipeProvider is given a URL and should return a Source or Target pipe.

type Provider 

type Provider interface {
	Schema() []string
}

type S3Provider 

type S3Provider struct {
	Endpoint string
	Region   string
}

Provides files from an S3-like HTTP interface

func (S3Provider) Read 

func (p S3Provider) Read(u *url.URL) (io.ReadCloser, error)

func (S3Provider) Schema 

func (S3Provider) Schema() []string

func (S3Provider) Session 

func (p S3Provider) Session(u *url.URL) (*session.Session, error)

func (S3Provider) Write 

func (p S3Provider) Write(u *url.URL) (WriteDestroyCloser, error)

type S3PutObject 

type S3PutObject struct {
	// contains filtered or unexported fields
}

func (*S3PutObject) Close 

func (o *S3PutObject) Close() error

func (*S3PutObject) Destroy 

func (o *S3PutObject) Destroy() error

func (*S3PutObject) Write 

func (o *S3PutObject) Write(b []byte) (int, error)

type SourcePipe 

type SourcePipe struct {
	Name   string
	Path   string
	URL    *url.URL
	Stream io.ReadCloser
}

type SourceProvider 

type SourceProvider interface {
	Provider
	Read(*url.URL) (io.ReadCloser, error)
}

type Sources 

type Sources []*SourcePipe

func (Sources) Copy 

func (s Sources) Copy(ctx context.Context) error

Copy starts copying data from each source stream to each existing named pipe

func (Sources) Teardown 

func (s Sources) Teardown() (mu *MultiError)

type TargetPipe 

type TargetPipe struct {
	Name   string
	Path   string
	URL    *url.URL
	Stream WriteDestroyCloser
}

type TargetProvider 

type TargetProvider interface {
	Provider
	Write(*url.URL) (WriteDestroyCloser, error)
}

type Targets 

type Targets []*TargetPipe

func (Targets) Copy 

func (t Targets) Copy(ctx context.Context) error

func (Targets) Teardown 

func (t Targets) Teardown() (mu *MultiError)

type Task 

type Task struct {
	Call Call
	// Preserve created target objects on failure
	Preserve       bool
	MountDirectory string
	Providers      []Provider

	// Sources provides a mapping of directory local named pipes to their equivalent URL
	Sources UrlMapping
	Targets UrlMapping

	Stdin *Url

	// Stdout is the target URL (if defined) for the output of the command
	Stdout *Url
	Stderr *Url
}

func (*Task) SetupInput 

func (t *Task) SetupInput() (io.ReadCloser, error)

func (*Task) SetupOutput 

func (t *Task) SetupOutput() (stdout WriteDestroyCloser, stderr WriteDestroyCloser, err error)

func (*Task) Source 

func (t *Task) Source(u *url.URL) (*SourcePipe, error)

func (*Task) Target 

func (t *Task) Target(u *url.URL) (*TargetPipe, error)

type TemplateGenerator 

type TemplateGenerator struct {
	Provider PipeProvider

	SourceTags UrlMapping
	Sources    Sources

	TargetTags UrlMapping
	Targets    Targets
}

A TemplateGenerator replaces command-line argument values with a real location of a fifo on the file-system

func (*TemplateGenerator) Replace 

func (g *TemplateGenerator) Replace(args []string) ([]string, error)

Replace replaces the contents of args with the templated values found from the given source and target mappings. returns all un-used source and target mappings.

type Url 

type Url url.URL

func (*Url) UnmarshalFlag 

func (f *Url) UnmarshalFlag(value string) error

type UrlMapping 

type UrlMapping map[string]*Url

func (UrlMapping) UnmarshalFlag 

func (m UrlMapping) UnmarshalFlag(value string) error

UnmarshalFlag implements un-marshalling a flag value into the URL mapping. Where the format is key:url

type WriteDestroyCloser 

type WriteDestroyCloser interface {
	io.WriteCloser
	// Teardown is called when the command fails, signalling that the object should be removed.
	// Closer must be closed before calling abort.
	Destroy() error
}

func ProvideTarget 

func ProvideTarget(u *url.URL, providers ...Provider) (WriteDestroyCloser, error)

Source Files

View all Source files

Directories

Path Synopsis
cmd
fifo

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.