Documentation
¶
Index ¶
Constants ¶
const ( AWSAccessKeyID = "AWSAccessKeyID" AWSSecretAccessKey = "AWSSecretAccessKey" S3Region = "S3Region" S3GlobalCannedACL = "S3GlobalCannedACL" S3SSEAlgorithm = "S3SSEAlgorithm" )
const ( SFTPUsername = "SFTPUsername" SFTPPassword = "SFTPPassword" SFTPPort = "SFTPPort" SFTPKey = "SFTPKey" SFTPDefaultPort = "22" )
Used for properties.Get.
const ( SSHErrFileNotFound = 2 SSHErrFileExists = 4 )
Redeclaring SSH error codes since the originals are not exported http://api.libssh.org/master/group__libssh__sftp.html#member-group NOTE(Azim): File exists errors are actually mapping to SSH_ERR_FAILURE (4) instead of file exists (11). Compensating for that since we need to minimally be able to tell when a file exists, although this may mask others.
const ( // Buffer roughly this much data in memory before flushing a multipart // chunk. (The true maximum multipart upload fragment size is much larger.) // Logistically, 1GiB chunks are more efficient, but we'd probably want to // spool to disk instead. This maximum spool size makes our maximum single // object file size 1TiB, as 10000 parts are allowed to any S3 object. MaxSpoolSizeBytes = 1024 * 1024 * 100 )
Variables ¶
var (
GCPServiceAccount = flag.String("gcpServiceAccount",
"/var/run/secrets/google-service-account/key.json",
"Location to Google Service Account JSON file")
)
Functions ¶
This section is empty.
Types ¶
type BaseEndpoint ¶
type BaseEndpoint struct {
Name string `json:"name"`
Type string `json:"type"`
PermissionTestFilename string `json:"permission_test_filename"`
}
BaseEndpoint provides common fields for all endpoints. Though it currently only contains a |Name| field, it's important to maintain this inheritence to allow us to use |Name| as a primary key in the endpoint namespace.
func (*BaseEndpoint) Validate ¶
func (ep *BaseEndpoint) Validate() error
Validate satisfies the Model interface from model-builder. Endpoint implementations are built from SQL, and Validate()'d as they're ETL'd into etcd.
type Endpoint ¶
type Endpoint interface {
// CheckPermissions connects to the endpoint and confirms that the
// passed credentials have read/write permissions in the root directory.
CheckPermissions() error
// Connect returns a FileSystem to be used by the caller, allowing the caller
// to specify an arbitrary set of additional |Properties|. In most cases,
// this will be unnecessary, as all connection details will be specified
// by the Endpoint. |Properties| passed will be merged with those defined in the
// Endpoint, overwriting the Endpoint properties where necessary.
Connect(Properties) (FileSystem, error)
// Validate inspects the endpoint and confirms that all internal fields are
// well-formed. Also satisfies the Model interface.
Validate() error
}
Endpoint reflects a common interface for structs with connection information to an arbitrary |FileSystem|.
func UnmarshalEndpoint ¶
UnmarshalEndpoint takes a byte array of json data (usually from etcd) and returns the appropriate |Endpoint| interface implementation.
type Endpoint_DEPRECATED ¶
type Endpoint_DEPRECATED struct {
// AWS
AWSAccessKeyID string `json:"aws_access_key_id"`
AWSSecretAccessKey string `json:"aws_secret_access_key"`
S3GlobalCannedACL string `json:"s3_global_canned_acl"`
S3Region string `json:"s3_region"`
S3Bucket string `json:"s3_bucket"`
S3Subfolder string `json:"s3_subfolder"`
// TODO(joshk): Migrate this to 's3_sse_algorithm'.
S3SSEAlgorithm string `json:"sse"`
// SFTP
SFTPHostname string `json:"sftp_hostname"`
// TODO(joshk): This should be an integer.
SFTPPort string `json:"sftp_port"`
SFTPUsername string `json:"sftp_username"`
SFTPPassword string `json:"sftp_password"`
SFTPDirectory string `json:"sftp_directory"`
}
Endpoint_DEPRECATED is the old-style partner-endpoint, to be replaced by the Endpoint interface.
func (*Endpoint_DEPRECATED) IsS3 ¶
func (ep *Endpoint_DEPRECATED) IsS3() bool
IsS3 returns whether or not the config describes an S3 endpoint.
func (*Endpoint_DEPRECATED) IsSFTP ¶
func (ep *Endpoint_DEPRECATED) IsSFTP() bool
IsSFTP returns whether or not the config describes an SFTP endpoint.
func (*Endpoint_DEPRECATED) Properties ¶
func (ep *Endpoint_DEPRECATED) Properties(keyPath string) Properties
Properties returns a cloudstore.Properties map for the given Endpoint.
func (*Endpoint_DEPRECATED) Subfolder ¶
func (ep *Endpoint_DEPRECATED) Subfolder() string
Subfolder returns the value of the directory beyond the root to upload a file to.
func (*Endpoint_DEPRECATED) URI ¶
func (ep *Endpoint_DEPRECATED) URI() string
URI returns a fully qualified URI string for the given endpoint .
func (*Endpoint_DEPRECATED) Validate ¶
func (ep *Endpoint_DEPRECATED) Validate() error
Validate satisfies the model interface
type File ¶
type File interface {
http.File
io.Writer
// ContentSignature is a representation of the file's data, ideally
// a content sum or ETag (in the case of cloud storage providers).
// Calling this should not require a calculation that reads the whole file.
ContentSignature() (string, error)
}
File extends the read-only http.File interface with an io.Writer.
type FileSystem ¶
type FileSystem interface {
http.FileSystem
// Releases the FileSystem and associated resources.
Close() error
// Writes |to| by directly copying from |from|. Iff an error is encountered
// (either via |to.Write()| or *|from.Read()|*), the partially-written content
// is removed or never made observable on the target FileSystem (depending on
// provider semantics). Otherwise, |to| is visible on the FileSystem after
// the call completes. In all cases, |to| is invalidated (eg, Close()d)
// after this call. Re-tryable bulk transfers should generally use
// this method for all-or-nothing behavior.
CopyAtomic(to File, from io.Reader) (n int64, err error)
// Creates a directory |path|, along with any necessary parents.
MkdirAll(name string, perm os.FileMode) error
// Generalized open call. It opens the named file with the specified |flag|
// and |perm|. For cloud file systems, |flag| and |perm| are interpreted and
// mapped into the capabilities of the file system, and may be ignored.
OpenFile(name string, flag int, perm os.FileMode) (File, error)
// Indicates whether |ToURL| produces a authorized URL.
ProducesAuthorizedURL() bool
// Removes the named file or directory.
Remove(name string) error
// Produces a URL which fully identifies the resource. Depending on the
// provider, the URL should implicitly authorize the bearer for operation
// |method| within |validFor| duration.
ToURL(name, method string, validFor time.Duration) (*url.URL, error)
// Similar to |filepath.Walk|, calls a |filepath.WalkFunc| for every
// file or directory under the given |prefix|. Note that not all filesystems
// surface the concept of directories, and minimally, each driver only
// guarantees to return a recursive listing of files.
Walk(root string, walkFn filepath.WalkFunc) error
}
FileSystem extends the read-only methods of http.FileSystem with methods capable of writing files, making directories, removing files or directories, and for producing "signed" URLs granting the URL bearer time-limited access rights to the file. Throughout the interface, returned errors are mapped into os-pkg errors testable with os.IsNotExist(), os.IsExist(), etc.
func NewFileSystem ¶
func NewFileSystem(properties Properties, rawURL string) (FileSystem, error)
Selects a FileSystem implementation from |rawURL|. Implementations are determined by URL scheme, and the path roots the resulting FileSystem. Depending on provider, options are passed as URL query arguments.
func NewTmpFileSystem ¶
func NewTmpFileSystem() FileSystem
Returns a FileSystem backed by a new temporary directory. The returned FileSystem should be Close()d after use to clear temporary files.
type GCSEndpoint ¶
type GCSEndpoint struct {
BaseEndpoint
GCSBucket string `json:"bucket"`
GCSSubfolder string `json:"subfolder"`
}
GCSEndpoint is a fully-defined GCS endpoint with bucket and subfolder.
func (*GCSEndpoint) CheckPermissions ¶
func (ep *GCSEndpoint) CheckPermissions() error
CheckPermissions satisfies the Endpoint interface.
func (*GCSEndpoint) Connect ¶
func (ep *GCSEndpoint) Connect(more Properties) (FileSystem, error)
Connect satisfies the Endpoint interface, returning a usable connection to the underlying GCS filesystem.
func (*GCSEndpoint) Validate ¶
func (ep *GCSEndpoint) Validate() error
Validate satisfies the model interface.
type MapProperties ¶
MapProperties is a simple implementation of Properties backed by an in-memory map.
func (MapProperties) Get ¶
func (mp MapProperties) Get(key string) string
Get satisfies the |Properties| interface.
type Properties ¶
Properties gets values for keys, and allows a Filesytem to configure itself.
func EmptyProperties ¶
func EmptyProperties() Properties
EmptyProperties returns an empty set of properties, useful for callers who don't need to specify any additional connection parameters when initializing a FileSystem.
func LocationFromEndpoint ¶
func LocationFromEndpoint(keysAPI etcd.KeysAPI, path, keyPath string) (string, Properties)
LocationFromEndpoint returns a URI and properties given a partner-endpoints-style path in etcd. Optionally, |keyPath| can be attached to SFTP authentication.
func PropertiesFromFile ¶
func PropertiesFromFile(path string) Properties
type S3Endpoint ¶
type S3Endpoint struct {
BaseEndpoint
AWSAccessKeyID string `json:"access_key_id"`
AWSSecretAccessKey string `json:"secret_access_key"`
S3GlobalCannedACL string `json:"global_canned_acl"`
S3Region string `json:"region"`
S3Bucket string `json:"bucket"`
S3Subfolder string `json:"subfolder"`
S3SSEAlgorithm string `json:"sse_algorithm"`
}
S3Endpoint is a fully-defined S3 endpoint with bucket and subfolder.
func (*S3Endpoint) CheckPermissions ¶
func (ep *S3Endpoint) CheckPermissions() error
CheckPermissions satisfies the Endpoint interface.
func (*S3Endpoint) Connect ¶
func (ep *S3Endpoint) Connect(more Properties) (FileSystem, error)
Connect satisfies the Endpoint interface, returning a usable connection to the underlying S3 filesystem.
func (*S3Endpoint) Validate ¶
func (ep *S3Endpoint) Validate() error
Validate satisfies the model interface.
type S3Properties ¶
func (S3Properties) Get ¶
func (s S3Properties) Get(key string) string
type SFTPEndpoint ¶
type SFTPEndpoint struct {
BaseEndpoint
SFTPHostname string `json:"hostname"`
SFTPPort string `json:"port"`
SFTPUsername string `json:"username"`
SFTPPassword string `json:"password"`
SFTPDirectory string `json:"directory"`
SFTPKey string `json:"ssh_key"`
}
SFTPEndpoint is a fully-defined SFTP endpoint with subfolder.
func (*SFTPEndpoint) CheckPermissions ¶
func (ep *SFTPEndpoint) CheckPermissions() error
CheckPermissions satisfies the Endpoint interface.
func (*SFTPEndpoint) Connect ¶
func (ep *SFTPEndpoint) Connect(more Properties) (FileSystem, error)
Connect satisfies the Endpoint interface, returning a usable connection to the underlying SFTP filesystem.
func (*SFTPEndpoint) Validate ¶
func (ep *SFTPEndpoint) Validate() error
Validate satisfies the model interface.
type SSHConnDialer ¶
SSHConnDialer is used to set up the ssh connection by the sftp fs client.
var DefaultSSHConnDialer SSHConnDialer = new(net.Dialer)
DefaultSSHConnDialer allows users to configure their own ssh connection dialer for a more complex ssh connection used for setting up the sftp fs client.
Source Files