apiclient

Documentation

Index

• Variables
• func Simple(scraperDirectory string, clientServerURL string, environment map[string]string, ...) error
• func SimpleConnect(runID string, scraperDirectory string, clientServerURL string, ...) error
• func SimpleStart(runID string, scraperDirectory string, clientServerURL string, ...) error
• type Client
  • func New(url string) *Client
  • func (client *Client) CreateRun(options protocol.CreateRunOptions) (RunInterface, error)
  • func (client *Client) Hello() (hello protocol.Hello, err error)
• type EventIterator
  • func (iterator *EventIterator) More() bool
  • func (iterator *EventIterator) Next() (event protocol.Event, err error)
• type Run
  • func (run *Run) CreateEvent(event protocol.Event) (int, error)
  • func (run *Run) CreateFinishEvent(stage string, exitData protocol.ExitDataStage) (int, error)
  • func (run *Run) CreateFirstEvent() (int, error)
  • func (run *Run) CreateLastEvent() (int, error)
  • func (run *Run) CreateLogEvent(stage string, stream string, text string) (int, error)
  • func (run *Run) CreateStartEvent(stage string) (int, error)
  • func (run *Run) Delete() error
  • func (run *Run) GetApp() (io.ReadCloser, error)
  • func (run *Run) GetAppToDirectory(dir string) error
  • func (run *Run) GetCache() (io.ReadCloser, error)
  • func (run *Run) GetCacheToDirectory(dir string) error
  • func (run *Run) GetCacheToFile(path string) error
  • func (run *Run) GetEvents(lastID string) (*EventIterator, error)
  • func (run *Run) GetExitData() (exitData protocol.ExitData, err error)
  • func (run *Run) GetID() string
  • func (run *Run) GetOutput() (io.ReadCloser, error)
  • func (run *Run) GetOutputToFile(path string) error
  • func (run *Run) PutApp(appData io.Reader) error
  • func (run *Run) PutAppFromDirectory(dir string, ignorePaths []string) error
  • func (run *Run) PutCache(data io.Reader) error
  • func (run *Run) PutCacheFromDirectory(dir string) error
  • func (run *Run) PutOutput(data io.Reader) error
  • func (run *Run) PutOutputFromFile(path string) error
  • func (run *Run) Start(options *protocol.StartRunOptions) error
• type RunInterface

Variables

var ErrNotFound = errors.New("Not Found")

ErrNotFound corresponds to a 404

var ErrUnauthorized = errors.New("Unauthorized")

ErrUnauthorized corresponds to a 401

Functions

func Simple

func Simple(scraperDirectory string, clientServerURL string, environment map[string]string,
	outputFile string, cache bool, callbackURL string, apiKey string, eventCallback func(event protocol.Event) error, showProgress bool) error

Simple is a super simple high level way of running a scraper that exists on the local file system giving you a local callback for every event (including logs). This is used by the command line client It makes a simple common use case a little simpler to implement

func SimpleConnect

func SimpleConnect(runID string, scraperDirectory string, clientServerURL string, outputFile string, cache bool, eventCallback func(event protocol.Event) error, showProgress bool) error

SimpleConnect connects to a run that has been started and handles the rest

func SimpleStart

func SimpleStart(runID string, scraperDirectory string, clientServerURL string, environment map[string]string,
	outputFile string, cache bool, callbackURL string, showProgress bool) error

Types

type Client

type Client struct {
	URL        string
	HTTPClient *http.Client
}

Client is used to access the API

func New

func New(url string) *Client

New configures a new Client

func (*Client) CreateRun

func (client *Client) CreateRun(options protocol.CreateRunOptions) (RunInterface, error)

CreateRun is the first thing called. It creates a run

func (*Client) Hello

func (client *Client) Hello() (hello protocol.Hello, err error)

Hello does a simple ping type request to the API

type EventIterator

type EventIterator struct {
	// contains filtered or unexported fields
}

EventIterator is a stream of events

func (*EventIterator) More

func (iterator *EventIterator) More() bool

More checks whether another event is available

func (*EventIterator) Next

func (iterator *EventIterator) Next() (event protocol.Event, err error)

Next returns the next event

type Run

type Run struct {
	protocol.Run
	// Ignore this field when converting from/to json
	Client *Client
}

Run is what you get when you create a run and what you need to update it

func (*Run) CreateEvent

func (run *Run) CreateEvent(event protocol.Event) (int, error)

CreateEvent sends an event and returns an approximation of the number of bytes sent

func (*Run) CreateFinishEvent

func (run *Run) CreateFinishEvent(stage string, exitData protocol.ExitDataStage) (int, error)

CreateFinishEvent creates and sends a "finish" event

func (*Run) CreateFirstEvent

func (run *Run) CreateFirstEvent() (int, error)

CreateFirstEvent creates and sends a "first" event

func (*Run) CreateLastEvent

func (run *Run) CreateLastEvent() (int, error)

CreateLastEvent creates and sends a "last" event

func (*Run) CreateLogEvent

func (run *Run) CreateLogEvent(stage string, stream string, text string) (int, error)

CreateLogEvent creates and sends a "log" event

func (*Run) CreateStartEvent

func (run *Run) CreateStartEvent(stage string) (int, error)

CreateStartEvent creates and sends a "start" event

func (*Run) Delete

func (run *Run) Delete() error

Delete cleans up after a run is complete

func (*Run) GetApp

func (run *Run) GetApp() (io.ReadCloser, error)

GetApp downloads the tarred & gzipped scraper code

func (*Run) GetAppToDirectory

func (run *Run) GetAppToDirectory(dir string) error

GetAppToDirectory downloads the scraper code into a pre-existing directory on the filesystem

func (*Run) GetCache

func (run *Run) GetCache() (io.ReadCloser, error)

GetCache downloads the tarred & gzipped build cache

func (*Run) GetCacheToDirectory

func (run *Run) GetCacheToDirectory(dir string) error

GetCacheToDirectory downloads the cache into a pre-existing directory on the filesystem

func (*Run) GetCacheToFile

func (run *Run) GetCacheToFile(path string) error

GetCacheToFile downloads the cache (as a tar & gzipped file) and saves it (without uncompressing it)

func (*Run) GetEvents

func (run *Run) GetEvents(lastID string) (*EventIterator, error)

GetEvents returns a stream of events from the API If lastID is empty ("") then the stream starts from the beginning. Otherwise it starts from the first event after the one with the given ID.

func (*Run) GetExitData

func (run *Run) GetExitData() (exitData protocol.ExitData, err error)

GetExitData gets data about resource usage after everything has finished

func (*Run) GetID

func (run *Run) GetID() string

GetID returns the name of the run

func (*Run) GetOutput

func (run *Run) GetOutput() (io.ReadCloser, error)

GetOutput downloads the output of the run. Could be any file in any format.

func (*Run) GetOutputToFile

func (run *Run) GetOutputToFile(path string) error

GetOutputToFile downloads the output of the run and saves it in a file which it will create or overwrite.

func (*Run) PutApp

func (run *Run) PutApp(appData io.Reader) error

PutApp uploads the tarred & gzipped scraper code

func (*Run) PutAppFromDirectory

func (run *Run) PutAppFromDirectory(dir string, ignorePaths []string) error

PutAppFromDirectory uploads the scraper code from a directory on the filesystem ignorePaths is a list of paths (relative to dir) that should be ignored and not uploaded

func (*Run) PutCache

func (run *Run) PutCache(data io.Reader) error

PutCache uploads the tarred & gzipped build cache

func (*Run) PutCacheFromDirectory

func (run *Run) PutCacheFromDirectory(dir string) error

PutCacheFromDirectory uploads the cache from a directory on the filesystem

func (*Run) PutOutput

func (run *Run) PutOutput(data io.Reader) error

PutOutput uploads the output of the scraper

func (*Run) PutOutputFromFile

func (run *Run) PutOutputFromFile(path string) error

PutOutputFromFile uploads the contents of a file as the output of the scraper

func (*Run) Start

func (run *Run) Start(options *protocol.StartRunOptions) error

Start starts a run that has earlier been created TODO: Add setting of environment variables

type RunInterface

type RunInterface interface {
	GetID() string
	GetApp() (io.ReadCloser, error)
	GetCache() (io.ReadCloser, error)
	GetOutput() (io.ReadCloser, error)
	GetExitData() (exitData protocol.ExitData, err error)
	PutApp(data io.Reader) error
	PutCache(data io.Reader) error
	PutOutput(data io.Reader) error
	Start(options *protocol.StartRunOptions) error
	GetEvents(lastID string) (*EventIterator, error)
	CreateEvent(event protocol.Event) (int, error)
	Delete() error
	// The following methods operate on to top of the lower level methods above
	// TODO: Should the following methods be in a separate interface?
	GetAppToDirectory(dir string) error
	PutAppFromDirectory(dir string, ignorePaths []string) error
	GetCacheToFile(path string) error
	GetCacheToDirectory(dir string) error
	PutCacheFromDirectory(dir string) error
	GetOutputToFile(path string) error
	PutOutputFromFile(path string) error
	CreateStartEvent(stage string) (int, error)
	CreateFinishEvent(stage string, exitData protocol.ExitDataStage) (int, error)
	CreateLogEvent(stage string, stream string, text string) (int, error)
	CreateFirstEvent() (int, error)
	CreateLastEvent() (int, error)
}

RunInterface is the interface to interact with existing runs