stashcp

README

Stashcp Go Client

Building

Download the repo and cd into the directory. Build with the command:

$ make

It will make two executables. A static x86 executable, and an executable native to the building machine.

Testing

Run this simple command to download a test file

$ ./stashcp /osgconnect/public/dweitzel/blast/queries/query1 ./

Configuration

stashcp is affected by the environment variables:

Environment Variable	Description
OSG_DISABLE_PROXY_FALLBACK	Do not disable using proxies. By default, stashcp will try to use an HTTP proxy when connecting to a cache. If this environment variable is set (no value necessary, only if it's set), then stashcp will not fallback to no proxy if the proxy download fails.
STASHCP_MINIMUM_DOWNLOAD_SPEED	The lower limit a download will be cancelled, in bytes per second
STASH_NAMESPACE_URL	The URL to download the namespace and cache information. Default: https://topology.opensciencegrid.org/stashcache/namespaces

Documentation

Index

• Variables
• func AddError(err error) bool
• func ByteCountSI(b int64) string
• func DoStashCPSingle(sourceFile string, destination string, methods []string, recursive bool) (int64, error)
• func DownloadHTTP(transfer TransferDetails, dest string, token string) (int64, error)
• func ErrorsRetryable() bool
• func Find(slice []string, val string) (int, bool)
• func GetBestStashcache(cacheListName string) ([]string, error)
• func GetErrors() string
• func HasPort(host string) bool
• func IsDir(dirUrl *url.URL, token string, namespace Namespace) (bool, error)
• func IsRetryable(err error) bool
• func UploadFile(src string, dest *url.URL, token string, namespace Namespace) (int64, error)
• type Cache
• type ConnectionSetupError
  • func (e *ConnectionSetupError) Error() string
  • func (e *ConnectionSetupError) Is(target error) bool
  • func (e *ConnectionSetupError) Unwrap() error
• type Namespace
  • func GetNamespaces() ([]Namespace, error)
  • func MatchNamespace(path string) (Namespace, error)
  • func (ns *Namespace) GetCacheHosts() []string
  • func (ns *Namespace) GetCaches() []Cache
  • func (ns *Namespace) MatchCaches(caches []string) []Cache
• type NamespaceFull
• type OptionsStruct
• type ProgressReader
  • func (pr *ProgressReader) Close() error
  • func (pr *ProgressReader) Read(p []byte) (n int, err error)
• type SlowTransferError
  • func (e *SlowTransferError) Error() string
  • func (e *SlowTransferError) Is(target error) bool
• type TransferDetails
  • func NewTransferDetails(cache Cache, https bool) []TransferDetails
• type TransferResults

Variables

var CacheOverride bool

CacheOverride

var CachesJsonLocation string

var NearestCache string

Nearest cache

var NearestCacheList []string

List of caches, in order from closest to furthest

Functions

func AddError

func AddError(err error) bool

AddError will add an accumulated error to the error stack

func ByteCountSI

func ByteCountSI(b int64) string

func DoStashCPSingle

func DoStashCPSingle(sourceFile string, destination string, methods []string, recursive bool) (int64, error)

Start the transfer, whether read or write back

func DownloadHTTP

func DownloadHTTP(transfer TransferDetails, dest string, token string) (int64, error)

DownloadHTTP - Perform the actual download of the file

func ErrorsRetryable

func ErrorsRetryable() bool

ErrorsRetryable returns if the errors in the stack are retryable later

func Find

func Find(slice []string, val string) (int, bool)

Find takes a slice and looks for an element in it. If found it will return it's key, otherwise it will return -1 and a bool of false. From https://golangcode.com/check-if-element-exists-in-slice/

func GetBestStashcache

func GetBestStashcache(cacheListName string) ([]string, error)

func GetErrors

func GetErrors() string

func HasPort

func HasPort(host string) bool

HasPort test the host if it includes a port

func IsDir

func IsDir(dirUrl *url.URL, token string, namespace Namespace) (bool, error)

func IsRetryable

func IsRetryable(err error) bool

IsRetryable will return true if the error is retryable

func UploadFile

func UploadFile(src string, dest *url.URL, token string, namespace Namespace) (int64, error)

UploadFile Uploads a file using HTTP

Types

type Cache

type Cache struct {
	AuthEndpoint string `json:"auth_endpoint"`
	Endpoint     string `json:"endpoint"`
	Resource     string `json:"resource"`
}

Cache

type ConnectionSetupError

type ConnectionSetupError struct {
	URL string
	Err error
}

ConnectionSetupError is an error that is returned when a connection to the remote server fails

func (*ConnectionSetupError) Error

func (e *ConnectionSetupError) Error() string

func (*ConnectionSetupError) Is

func (e *ConnectionSetupError) Is(target error) bool

func (*ConnectionSetupError) Unwrap

func (e *ConnectionSetupError) Unwrap() error

type Namespace

type Namespace struct {
	Caches         []Cache `json:"caches"`
	Path           string  `json:"path"`
	ReadHTTPS      bool    `json:"readhttps"`
	UseTokenOnRead bool    `json:"usetokenonread"`
	WriteBackHost  string  `json:"writebackhost"`
	DirListHost    string  `json:"dirlisthost"`
}

Namespace holds the structure of stash namespaces

func GetNamespaces

func GetNamespaces() ([]Namespace, error)

GetNamespaces returns the list of namespaces

func MatchNamespace

func MatchNamespace(path string) (Namespace, error)

MatchNamespace matches the namespace passed in to the namespaces in the list

func (*Namespace) GetCacheHosts

func (ns *Namespace) GetCacheHosts() []string

func (*Namespace) GetCaches

func (ns *Namespace) GetCaches() []Cache

GetCaches returns the list of caches for the namespace

func (*Namespace) MatchCaches

func (ns *Namespace) MatchCaches(caches []string) []Cache

MatchCaches compares the caches passed in (presumably from an ordered list of caches) to the caches for the namespace, and returns the intersection of the two

type NamespaceFull

type NamespaceFull struct {
	Caches     []Cache     `json:"caches"`
	Namespaces []Namespace `json:"namespaces"`
}

type OptionsStruct

type OptionsStruct struct {
	ProgressBars bool
	Recursive    bool
	Token        string
}

var Options OptionsStruct

type ProgressReader

type ProgressReader struct {
	// contains filtered or unexported fields
}

ProgressReader wraps the io.Reader to get progress Adapted from https://stackoverflow.com/questions/26050380/go-tracking-post-request-progress

func (*ProgressReader) Close

func (pr *ProgressReader) Close() error

Close implments the close function of io.Closer

func (*ProgressReader) Read

func (pr *ProgressReader) Read(p []byte) (n int, err error)

Read implements the common read function for io.Reader

type SlowTransferError

type SlowTransferError struct {
	BytesTransferred int64
	BytesPerSecond   int64
	BytesTotal       int64
	Duration         time.Duration
}

SlowTransferError is an error that is returned when a transfer takes longer than the configured timeout

func (*SlowTransferError) Error

func (e *SlowTransferError) Error() string

func (*SlowTransferError) Is

func (e *SlowTransferError) Is(target error) bool

type TransferDetails

type TransferDetails struct {
	// Url is the url.URL of the cache and port
	Url url.URL

	// Proxy specifies if a proxy should be used
	Proxy bool
}

func NewTransferDetails

func NewTransferDetails(cache Cache, https bool) []TransferDetails

NewTransferDetails creates the TransferDetails struct with the given cache

type TransferResults

type TransferResults struct {
	Error      error
	Downloaded int64
}