core

package
v0.9.36 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Apr 17, 2017 License: BSD-2-Clause Imports: 33 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	CLIENT_STAT_ACTIVE_BUSY = "active-busy"
	CLIENT_STAT_ACTIVE_IDLE = "active-idle"
	CLIENT_STAT_SUSPEND     = "suspend"
	CLIENT_STAT_DELETED     = "deleted"
)
View Source 
const (
	JOB_STAT_INIT       = "init"
	JOB_STAT_QUEUED     = "queued"
	JOB_STAT_INPROGRESS = "in-progress"
	JOB_STAT_COMPLETED  = "completed"
	JOB_STAT_SUSPEND    = "suspend"
	JOB_STAT_DELETED    = "deleted"
)
View Source 
const (
	TASK_STAT_INIT       = "init"
	TASK_STAT_QUEUED     = "queued"
	TASK_STAT_INPROGRESS = "in-progress"
	TASK_STAT_PENDING    = "pending"
	TASK_STAT_SUSPEND    = "suspend"
	TASK_STAT_COMPLETED  = "completed"
	TASK_STAT_SKIPPED    = "user_skipped"
	TASK_STAT_FAIL_SKIP  = "skipped"
	TASK_STAT_PASSED     = "passed"
)
View Source 
const (
	WORK_STAT_INIT        = "init"     //initial state
	WORK_STAT_QUEUED      = "queued"   // . also: after requeue ; after failures below max ; on WorkQueue.Add()
	WORK_STAT_CHECKOUT    = "checkout" // normal work checkout ; client registers that already has a workunit (e.g. after reboot of server)
	WORK_STAT_SUSPEND     = "suspend"  // on MAX_FAILURE ; on SuspendJob
	WORK_STAT_DONE        = "done"     // client-side, done.
	WORK_STAT_FAIL        = "fail"     // client-side, workunit computation or IO error
	WORK_STAT_PREPARED    = "prepared" // client-side, after argument parsing
	WORK_STAT_COMPUTED    = "computed" // client-side, after computation is done, before upload
	WORK_STAT_DISCARDED   = "discarded"
	WORK_STAT_PROXYQUEUED = "proxyqueued"
)

Variables

View Source 
var (
	QMgr          ResourceMgr
	Service       string = "unknown"
	Self          *Client
	ProxyWorkChan chan bool
	Server_UUID   string
	JM            *JobMap
)
View Source 
var (
	CGNameRegex = regexp.MustCompile(`^[A-Za-z0-9\_\-\.]+$`)
)
View Source 
var DocumentMaxByte = 16777216

mongodb has hard limit of 16 MB docuemnt size

View Source 
var JOB_STATS_ACTIVE = []string{JOB_STAT_QUEUED, JOB_STAT_INPROGRESS}
View Source 
var JOB_STATS_REGISTERED = []string{JOB_STAT_QUEUED, JOB_STAT_INPROGRESS, JOB_STAT_SUSPEND}
View Source 
var JOB_STATS_TO_RECOVER = []string{JOB_STAT_INIT, JOB_STAT_QUEUED, JOB_STAT_INPROGRESS, JOB_STAT_SUSPEND}
View Source 
var JobInfoIndexes = []string{"name", "submittime", "completedtime", "pipeline", "clientgroups", "project", "service", "user", "priority", "userattr.submission"}

indexed info fields for search

Functions

func DBGetJobAcl added in v0.9.33 

func DBGetJobAcl(job_id string) (_acl acl.Acl, err error)

func DbFindDistinct added in v0.9.26 

func DbFindDistinct(q bson.M, d string) (results interface{}, err error)

func DbUpdateJobField added in v0.9.33 

func DbUpdateJobField(job_id string, key string, value interface{}) (err error)

func DeleteClientGroup added in v0.9.3 

func DeleteClientGroup(id string) (err error)

func GetJobCount added in v0.9.12 

func GetJobCount(q bson.M) (count int, err error)

func GetJobIdByTaskId 

func GetJobIdByTaskId(taskid string) (jobid string, err error)

misc

func GetJobIdByWorkId 

func GetJobIdByWorkId(workid string) (jobid string, err error)

func GetTaskIdByWorkId 

func GetTaskIdByWorkId(workid string) (taskid string, err error)

func HasInfoField added in v0.9.26 

func HasInfoField(a string) bool

func InitAwfMgr 

func InitAwfMgr()

func InitClientGroupDB added in v0.9.3 

func InitClientGroupDB()

func InitJobDB 

func InitJobDB()

func InitProxyWorkChan 

func InitProxyWorkChan()

func InitReaper added in v0.9.26 

func InitReaper()

func InitResMgr 

func InitResMgr(service string)

func IsFirstTask 

func IsFirstTask(taskid string) bool

func NotifyWorkunitProcessed 

func NotifyWorkunitProcessed(work *Workunit, perf *WorkPerf) (err error)

functions for REST API communication (=deprecated=) notify AWE server a workunit is finished with status either "failed" or "done", and with perf statistics if "done"

func PostNode 

func PostNode(io *IO, numParts int) (nodeid string, err error)

Create a shock node for output (=deprecated=)

func PostNodeWithToken 

func PostNodeWithToken(io *IO, numParts int, token string) (nodeid string, err error)

func PushOutputData 

func PushOutputData(work *Workunit) (size int64, err error)

deprecated, see cache.UploadOutputData

func PutFileToShock 

func PutFileToShock(filename string, host string, nodeid string, rank int, token string, attrfile string, ntype string, formopts map[string]string, nodeattr map[string]interface{}) (err error)

func ReloadFromDisk 

func ReloadFromDisk(path string) (err error)

func SetClientProfile added in v0.9.33 

func SetClientProfile(profile *Client)

func ShockPutIndex 

func ShockPutIndex(host string, nodeid string, indexname string, token string) (err error)

func UpdateJobState 

func UpdateJobState(jobid string, newstate string, oldstates []string) (err error)

update job state to "newstate" only if the current state is in one of the "oldstates"

Types

type CQMgr 

type CQMgr struct {
	// contains filtered or unexported fields
}

this struct is embedded in ServerMgr

func NewCQMgr 

func NewCQMgr() *CQMgr

func (*CQMgr) AddClient added in v0.9.33 

func (qm *CQMgr) AddClient(client *Client, lock bool)

func (*CQMgr) CheckClient added in v0.9.33 

func (qm *CQMgr) CheckClient(client *Client) (ok bool, err error)

func (*CQMgr) CheckoutWorkunits 

func (qm *CQMgr) CheckoutWorkunits(req_policy string, client_id string, client *Client, available_bytes int64, num int) (workunits []*Workunit, err error)

func (*CQMgr) ClientChecker 

func (qm *CQMgr) ClientChecker()

func (*CQMgr) ClientHandle added in v0.9.16 

func (qm *CQMgr) ClientHandle()

func (*CQMgr) ClientHeartBeat 

func (qm *CQMgr) ClientHeartBeat(id string, cg *ClientGroup) (hbmsg HBmsg, err error)

func (*CQMgr) ClientIdStatusChange added in v0.9.33 

func (qm *CQMgr) ClientIdStatusChange(id string, new_status string, client_write_lock bool) (err error)

func (*CQMgr) ClientStatusChange added in v0.9.16 

func (qm *CQMgr) ClientStatusChange(client *Client, new_status string, client_write_lock bool) (err error)

func (*CQMgr) DeleteClient 

func (qm *CQMgr) DeleteClient(client *Client) (err error)

func (*CQMgr) DeleteClientById added in v0.9.33 

func (qm *CQMgr) DeleteClientById(id string) (err error)

func (*CQMgr) DeleteClientByUser added in v0.9.3 

func (qm *CQMgr) DeleteClientByUser(id string, u *user.User) (err error)

func (*CQMgr) DeleteClients added in v0.9.33 

func (qm *CQMgr) DeleteClients(delete_clients []string)

func (*CQMgr) EnqueueWorkunit 

func (qm *CQMgr) EnqueueWorkunit(work *Workunit) (err error)

func (*CQMgr) FetchDataToken 

func (qm *CQMgr) FetchDataToken(workid string, clientid string) (token string, err error)

func (*CQMgr) GetAllClientsByUser added in v0.9.3 

func (qm *CQMgr) GetAllClientsByUser(u *user.User) (clients []*Client, err error)

func (*CQMgr) GetClient 

func (qm *CQMgr) GetClient(id string, lock_clientmap bool) (client *Client, ok bool, err error)

func (*CQMgr) GetClientByUser added in v0.9.3 

func (qm *CQMgr) GetClientByUser(id string, u *user.User) (client *Client, err error)

func (*CQMgr) GetClientMap added in v0.9.33 

func (qm *CQMgr) GetClientMap() *ClientMap

func (*CQMgr) GetWorkById 

func (qm *CQMgr) GetWorkById(id string) (workunit *Workunit, err error)

func (*CQMgr) HasClient added in v0.9.16 

func (qm *CQMgr) HasClient(id string, lock_clientmap bool) (has bool, err error)

func (*CQMgr) ListClients added in v0.9.16 

func (qm *CQMgr) ListClients() (ids []string, err error)

func (*CQMgr) NotifyWorkStatus 

func (qm *CQMgr) NotifyWorkStatus(notice Notice)

func (*CQMgr) ReQueueWorkunitByClient 

func (qm *CQMgr) ReQueueWorkunitByClient(client *Client, client_write_lock bool) (err error)

func (*CQMgr) RegisterNewClient 

func (qm *CQMgr) RegisterNewClient(files FormFiles, cg *ClientGroup) (client *Client, err error)

func (*CQMgr) RemoveClient added in v0.9.16 

func (qm *CQMgr) RemoveClient(id string, lock bool)

lock is for clientmap

func (*CQMgr) ResumeClient 

func (qm *CQMgr) ResumeClient(id string) (err error)

func (*CQMgr) ResumeClientByUser added in v0.9.3 

func (qm *CQMgr) ResumeClientByUser(id string, u *user.User) (err error)

func (*CQMgr) ResumeSuspendedClients 

func (qm *CQMgr) ResumeSuspendedClients() (count int, err error)

func (*CQMgr) ResumeSuspendedClientsByUser added in v0.9.3 

func (qm *CQMgr) ResumeSuspendedClientsByUser(u *user.User) (count int)

func (*CQMgr) ShowWorkQueue 

func (qm *CQMgr) ShowWorkQueue()

show functions used in debug

func (*CQMgr) ShowWorkunits 

func (qm *CQMgr) ShowWorkunits(status string) (workunits []*Workunit, err error)

func (*CQMgr) ShowWorkunitsByUser added in v0.9.3 

func (qm *CQMgr) ShowWorkunitsByUser(status string, u *user.User) (workunits []*Workunit)

func (*CQMgr) SuspendAllClients 

func (qm *CQMgr) SuspendAllClients() (count int, err error)

func (*CQMgr) SuspendAllClientsByUser added in v0.9.3 

func (qm *CQMgr) SuspendAllClientsByUser(u *user.User) (count int)

func (*CQMgr) SuspendClient 

func (qm *CQMgr) SuspendClient(id string, client *Client, client_write_lock bool) (err error)

use id OR client

func (*CQMgr) SuspendClientByUser added in v0.9.3 

func (qm *CQMgr) SuspendClientByUser(id string, u *user.User) (err error)

func (*CQMgr) UpdateSubClients 

func (qm *CQMgr) UpdateSubClients(id string, count int) (err error)

func (*CQMgr) UpdateSubClientsByUser added in v0.9.3 

func (qm *CQMgr) UpdateSubClientsByUser(id string, count int, u *user.User)

type Client 

type Client struct {
	RWMutex
	Id              string          `bson:"id" json:"id"`
	Name            string          `bson:"name" json:"name"`
	Group           string          `bson:"group" json:"group"`
	User            string          `bson:"user" json:"user"`
	Domain          string          `bson:"domain" json:"domain"`
	InstanceId      string          `bson:"instance_id" json:"instance_id"`
	InstanceType    string          `bson:"instance_type" json:"instance_type"`
	Host            string          `bson:"host" json:"host"`
	CPUs            int             `bson:"cores" json:"cores"`
	Apps            []string        `bson:"apps" json:"apps"`
	RegTime         time.Time       `bson:"regtime" json:"regtime"`
	Serve_time      string          `bson:"serve_time" json:"serve_time"`
	Idle_time       int             `bson:"idle_time" json:"idle_time"`
	Status          string          `bson:"Status" json:"Status"`
	Total_checkout  int             `bson:"total_checkout" json:"total_checkout"`
	Total_completed int             `bson:"total_completed" json:"total_completed"`
	Total_failed    int             `bson:"total_failed" json:"total_failed"`
	Current_work    map[string]bool `bson:"current_work" json:"current_work"` // the bool in the mapping is deprecated. It used to indicate completed work that could not be returned to server
	Skip_work       []string        `bson:"skip_work" json:"skip_work"`
	Last_failed     int             `bson:"-" json:"-"`
	Tag             bool            `bson:"-" json:"-"`
	Proxy           bool            `bson:"proxy" json:"proxy"`
	SubClients      int             `bson:"subclients" json:"subclients"`
	GitCommitHash   string          `bson:"git_commit_hash" json:"git_commit_hash"`
	Version         string          `bson:"version" json:"version"`
	// contains filtered or unexported fields
}

func NewClient 

func NewClient() (client *Client)

func NewProfileClient 

func NewProfileClient(filepath string) (client *Client, err error)

create Client object from json file

func (*Client) Add_work added in v0.9.33 

func (cl *Client) Add_work(workid string) (err error)

lock always

func (*Client) Append_Skip_work added in v0.9.33 

func (cl *Client) Append_Skip_work(workid string, write_lock bool) (err error)

func (*Client) Contains_Skip_work_nolock added in v0.9.33 

func (cl *Client) Contains_Skip_work_nolock(workid string) (c bool)

func (*Client) Current_work_delete added in v0.9.33 

func (cl *Client) Current_work_delete(workid string, write_lock bool) (err error)

func (*Client) Current_work_length added in v0.9.33 

func (cl *Client) Current_work_length(lock bool) (clength int, err error)

func (*Client) Get_Ack added in v0.9.33 

func (cl *Client) Get_Ack() (ack CoAck, err error)

func (*Client) Get_Last_failed added in v0.9.33 

func (cl *Client) Get_Last_failed() (count int, err error)

func (*Client) Get_Status added in v0.9.33 

func (cl *Client) Get_Status(do_read_lock bool) (s string, err error)

func (*Client) Get_Total_checkout added in v0.9.33 

func (cl *Client) Get_Total_checkout() (count int, err error)

func (*Client) Get_Total_completed added in v0.9.33 

func (cl *Client) Get_Total_completed() (count int, err error)

func (*Client) Get_Total_failed added in v0.9.33 

func (cl *Client) Get_Total_failed() (count int, err error)

func (*Client) Get_current_work added in v0.9.33 

func (cl *Client) Get_current_work(do_read_lock bool) (current_work_ids []string, err error)

func (*Client) Increment_last_failed added in v0.9.33 

func (cl *Client) Increment_last_failed(err error)

func (*Client) Increment_total_checkout added in v0.9.33 

func (cl *Client) Increment_total_checkout(err error)

func (*Client) Increment_total_completed added in v0.9.33 

func (cl *Client) Increment_total_completed() (err error)

func (*Client) Increment_total_failed added in v0.9.33 

func (cl *Client) Increment_total_failed(write_lock bool) (err error)

func (*Client) Init added in v0.9.33 

func (client *Client) Init()

invoked by NewClient or manually after unmarshalling

func (*Client) IsBusy 

func (cl *Client) IsBusy(lock bool) (busy bool, err error)

func (*Client) Marshal added in v0.9.33 

func (cl *Client) Marshal() (result []byte, err error)

func (*Client) Set_Status added in v0.9.33 

func (cl *Client) Set_Status(s string, write_lock bool) (err error)

type ClientGroup added in v0.9.3 

type ClientGroup struct {
	Id           string                        `bson:"id" json:"id"`
	IP_CIDR      string                        `bson:"ip_cidr" json:"ip_cidr"`
	Name         string                        `bson:"name" json:"name"`
	Token        string                        `bson:"token" json:"token"`
	Acl          clientGroupAcl.ClientGroupAcl `bson:"acl" json:"-"`
	CreatedOn    time.Time                     `bson:"created_on" json:"created_on"`
	Expiration   time.Time                     `bson:"expiration" json:"expiration"`
	LastModified time.Time                     `bson:"last_modified" json:"last_modified"`
}

func CreateClientGroup added in v0.9.3 

func CreateClientGroup(name string, u *user.User) (cg *ClientGroup, err error)

func LoadClientGroup added in v0.9.3 

func LoadClientGroup(id string) (clientgroup *ClientGroup, err error)

func LoadClientGroupByName added in v0.9.3 

func LoadClientGroupByName(name string) (clientgroup *ClientGroup, err error)

func LoadClientGroupByToken added in v0.9.3 

func LoadClientGroupByToken(token string) (clientgroup *ClientGroup, err error)

func (*ClientGroup) Save added in v0.9.3 

func (cg *ClientGroup) Save() (err error)

func (*ClientGroup) SetToken added in v0.9.3 

func (cg *ClientGroup) SetToken()

type ClientGroups added in v0.9.3 

type ClientGroups []ClientGroup

ClientGroup array type

func (*ClientGroups) GetPaginated added in v0.9.3 

func (n *ClientGroups) GetPaginated(q bson.M, limit int, offset int, order string, direction string) (count int, err error)

type ClientMap added in v0.9.33 

type ClientMap struct {
	RWMutex
	// contains filtered or unexported fields
}

func NewClientMap added in v0.9.33 

func NewClientMap() *ClientMap

func (*ClientMap) Add added in v0.9.33 

func (cl *ClientMap) Add(client *Client, lock bool)

func (*ClientMap) Delete added in v0.9.33 

func (cl *ClientMap) Delete(client_id string, lock bool) (err error)

func (*ClientMap) Get added in v0.9.33 

func (cl *ClientMap) Get(client_id string, lock bool) (client *Client, ok bool, err error)

func (*ClientMap) GetClientIds added in v0.9.33 

func (cl *ClientMap) GetClientIds() (ids []string, err error)

func (*ClientMap) GetClients added in v0.9.33 

func (cl *ClientMap) GetClients() (clients []*Client, err error)

func (*ClientMap) Has added in v0.9.33 

func (cl *ClientMap) Has(client_id string, lock bool) (ok bool, err error)

type ClientMgr 

type ClientMgr interface {
	RegisterNewClient(FormFiles, *ClientGroup) (*Client, error)
	ClientHeartBeat(string, *ClientGroup) (HBmsg, error)
	GetClient(string, bool) (*Client, bool, error)
	GetClientByUser(string, *user.User) (*Client, error)
	//GetAllClients() []*Client
	GetClientMap() *ClientMap
	GetAllClientsByUser(*user.User) ([]*Client, error)
	DeleteClient(*Client) error
	DeleteClientById(string) error
	DeleteClientByUser(string, *user.User) error
	SuspendClient(string, *Client, bool) error
	SuspendClientByUser(string, *user.User) error
	ResumeClient(string) error
	ResumeClientByUser(string, *user.User) error
	ResumeSuspendedClients() (int, error)
	ResumeSuspendedClientsByUser(*user.User) int
	SuspendAllClients() (int, error)
	SuspendAllClientsByUser(*user.User) int
	ClientChecker()
	UpdateSubClients(string, int) error
	UpdateSubClientsByUser(string, int, *user.User)
}

type ClientWorkMgr 

type ClientWorkMgr interface {
	ClientMgr
	WorkMgr
}

type Clients added in v0.9.33 

type Clients []*Client

func (*Clients) RLockRecursive added in v0.9.33 

func (cs *Clients) RLockRecursive()

func (*Clients) RUnlockRecursive added in v0.9.33 

func (cs *Clients) RUnlockRecursive()

type CoAck 

type CoAck struct {
	// contains filtered or unexported fields
}

type CoReq 

type CoReq struct {
	// contains filtered or unexported fields
}

type Command 

type Command struct {
	Name          string   `bson:"name" json:"name"`
	Args          string   `bson:"args" json:"args"`
	Dockerimage   string   `bson:"Dockerimage" json:"Dockerimage"` // for Shock (TODO rename this !)
	DockerPull    string   `bson:"dockerPull" json:"dockerPull"`   // docker pull
	Cmd_script    []string `bson:"cmd_script" json:"cmd_script"`
	Environ       Envs     `bson:"environ" json:"environ"`
	HasPrivateEnv bool     `bson:"has_private_env" json:"has_private_env"`
	Description   string   `bson:"description" json:"description"`
	ParsedArgs    []string `bson:"-" json:"-"`
}

func NewCommand 

func NewCommand(name string) *Command

type Command_p added in v0.9.3 

type Command_p struct {
	Environ *Environ_p `json:"environ"`
}

type Environ_p added in v0.9.3 

type Environ_p struct {
	Private map[string]string `json:"private"`
}

following special code is in order to unmarshal the private field Command.Environ.Private, so put them in to this file for less confusion

type Envs added in v0.9.3 

type Envs struct {
	Public  map[string]string `bson:"public" json:"public"`
	Private map[string]string `bson:"private" json:"-"`
}

type FormFile 

type FormFile struct {
	Name     string
	Path     string
	Checksum map[string]string
}

type FormFiles 

type FormFiles map[string]FormFile

type HBmsg 

type HBmsg map[string]string //map[op]obj1,obj2 e.g. map[discard]=work1,work2

heartbeat response from awe-server to awe-client used for issue operation request to client, e.g. discard suspended workunits

type Helper added in v0.9.33 

type Helper struct {
	AWE_tasks *map[string]*Task
	// contains filtered or unexported fields
}

type IO 

type IO struct {
	FileName      string                 `bson:"filename" json:"filename"`
	Name          string                 `bson:"name" json:"name"`     // specifies abstract name of output as defined by the app
	AppPosition   int                    `bson:"appposition" json:"-"` // specifies position in app output array
	Directory     string                 `bson:"directory" json:"directory"`
	Host          string                 `bson:"host" json:"host"`
	Node          string                 `bson:"node" json:"node"`
	Url           string                 `bson:"url"  json:"url"` // can be shock or any other url
	Size          int64                  `bson:"size" json:"size"`
	MD5           string                 `bson:"md5" json:"-"`
	Cache         bool                   `bson:"cache" json:"cache"` // indicates that this files is "predata"" that needs to be cached
	Origin        string                 `bson:"origin" json:"origin"`
	Path          string                 `bson:"path" json:"-"`
	Optional      bool                   `bson:"optional" json:"-"`
	Nonzero       bool                   `bson:"nonzero"  json:"nonzero"`
	DataToken     string                 `bson:"datatoken"  json:"-"`
	Intermediate  bool                   `bson:"Intermediate"  json:"-"`
	Temporary     bool                   `bson:"temporary"  json:"temporary"`
	ShockFilename string                 `bson:"shockfilename" json:"shockfilename"`
	ShockIndex    string                 `bson:"shockindex" json:"shockindex"` // on input it indicates that Shock node has to be indexed by AWE server
	AttrFile      string                 `bson:"attrfile" json:"attrfile"`
	NoFile        bool                   `bson:"nofile" json:"nofile"`
	Delete        bool                   `bson:"delete" json:"delete"` // speficies that this is a temorary node, to be deleted from shock on job completion
	Type          string                 `bson:"type" json:"type"`
	NodeAttr      map[string]interface{} `bson:"nodeattr" json:"nodeattr"` // specifies attribute data to be stored in shock node (output only)
	FormOptions   map[string]string      `bson:"formoptions" json:"formoptions"`
	Uncompress    string                 `bson:"uncompress" json:"uncompress"` // tells AWE client to uncompress this file, e.g. "gzip"
}

func NewIO 

func NewIO() *IO

func (*IO) DataUrl 

func (io *IO) DataUrl() (dataurl string, err error)

func (*IO) DeleteNode added in v0.9.3 

func (io *IO) DeleteNode() (err error)

func (*IO) GetFileSize 

func (io *IO) GetFileSize() (size int64, modified bool, err error)

func (*IO) GetIndexInfo 

func (io *IO) GetIndexInfo() (idxinfo map[string]shock.IdxInfo, err error)

func (*IO) GetIndexUnits 

func (io *IO) GetIndexUnits(indextype string) (totalunits int, err error)

func (*IO) GetShockNode 

func (io *IO) GetShockNode() (node *shock.ShockNode, err error)

func (*IO) HasFile added in v0.9.16 

func (io *IO) HasFile() bool

func (*IO) TotalUnits 

func (io *IO) TotalUnits(indextype string) (count int, err error)

type IOmap 

type IOmap map[string]*IO // [filename]attributes

Deprecated JobDep struct uses deprecated TaskDep struct which uses the deprecated IOmap. Maintained for backwards compatibility. Jobs that cannot be parsed into the Job struct, but can be parsed into the JobDep struct will be translated to the new Job struct. (=deprecated=)

func NewIOmap 

func NewIOmap() IOmap

(=deprecated=)

func (IOmap) Add 

func (i IOmap) Add(name string, host string, node string, md5 string, cache bool)

(=deprecated=)

func (IOmap) Find 

func (i IOmap) Find(name string) *IO

(=deprecated=)

func (IOmap) Has 

func (i IOmap) Has(name string) bool

(=deprecated=)

type Info 

type Info struct {
	Name          string            `bson:"name" json:"name"`
	Xref          string            `bson:"xref" json:"xref"`
	Service       string            `bson:"service" json:"service"`
	Project       string            `bson:"project" json:"project"`
	User          string            `bson:"user" json:"user"`
	Pipeline      string            `bson:"pipeline" json:"pipeline"`
	ClientGroups  string            `bson:"clientgroups" json:"clientgroups"`
	SubmitTime    time.Time         `bson:"submittime" json:"submittime"`
	StartedTme    time.Time         `bson:"startedtime" json:"startedtime"`
	CompletedTime time.Time         `bson:"completedtime" json:"completedtime"`
	Priority      int               `bson:"priority" json:"priority"`
	Auth          bool              `bson:"auth" json:"auth"`
	DataToken     string            `bson:"datatoken" json:"-"`
	NoRetry       bool              `bson:"noretry" json:"noretry"`
	UserAttr      map[string]string `bson:"userattr" json:"userattr"`
	Description   string            `bson:"description" json:"description"`
	Tracking      bool              `bson:"tracking" json:"tracking"`
}

job info

func NewInfo 

func NewInfo() *Info

type Job 

type Job struct {
	JobRaw `bson:",inline"`
	Tasks  []*Task `bson:"tasks" json:"tasks"`
}

func CWL2AWE added in v0.9.33 

func CWL2AWE(_user *user.User, files FormFiles, cwl_workflow *cwl.Workflow, collection *cwl.CWL_collection) (job *Job, err error)

func CreateJobImport added in v0.9.26 

func CreateJobImport(u *user.User, file FormFile) (job *Job, err error)

func CreateJobUpload 

func CreateJobUpload(u *user.User, files FormFiles) (job *Job, err error)

func GetJob added in v0.9.33 

func GetJob(id string) (job *Job, err error)

func JobDepToJob added in v0.9.16 

func JobDepToJob(jobDep *JobDep) (job *Job, err error)

Takes the deprecated (version 1) Job struct and returns the version 2 Job struct or an error

func LoadJob 

func LoadJob(id string) (job *Job, err error)

func NewJob added in v0.9.33 

func NewJob() (job *Job)

func ParseJobTasksDep_DEPRECATED added in v0.9.36 

func ParseJobTasksDep_DEPRECATED(filename string) (job *Job, err error)

Parses job by job script using the deprecated Job struct. Maintained for backwards compatibility. (=deprecated=)

func ReadJobFile added in v0.9.33 

func ReadJobFile(filename string) (job *Job, err error)

func (*Job) Delete added in v0.9.26 

func (job *Job) Delete() (err error)

func (*Job) FilePath 

func (job *Job) FilePath() (path string, err error)

func (*Job) GetDataToken 

func (job *Job) GetDataToken() (token string)

func (*Job) GetJobLogs added in v0.9.27 

func (job *Job) GetJobLogs() (jlog *JobLog, err error)

func (*Job) GetPrivateEnv added in v0.9.3 

func (job *Job) GetPrivateEnv(taskid string) (env map[string]string)

func (*Job) GetRemainTasks added in v0.9.33 

func (job *Job) GetRemainTasks() (remain_tasks int, err error)

func (*Job) GetState added in v0.9.33 

func (job *Job) GetState(do_lock bool) (state string, err error)

func (*Job) GetTasks added in v0.9.33 

func (job *Job) GetTasks() (tasks []*Task, err error)

func (*Job) IncrementRemainTasks added in v0.9.33 

func (job *Job) IncrementRemainTasks(inc int, writelock bool) (err error)

func (*Job) IncrementResumed added in v0.9.33 

func (job *Job) IncrementResumed(inc int, writelock bool) (err error)

func (*Job) Init added in v0.9.33 

func (job *Job) Init() (changed bool, err error)

this has to be called after Unmarshalling from JSON

func (*Job) Mkdir 

func (job *Job) Mkdir() (err error)

func (*Job) NumTask 

func (job *Job) NumTask() int

func (*Job) Path 

func (job *Job) Path() (path string, err error)

---Path functions

func (*Job) RLockRecursive added in v0.9.33 

func (job *Job) RLockRecursive()

func (*Job) RUnlockRecursive added in v0.9.33 

func (job *Job) RUnlockRecursive()

func (*Job) Rmdir added in v0.9.26 

func (job *Job) Rmdir() (err error)

func (*Job) Save 

func (job *Job) Save() (err error)

func (*Job) SaveToDisk added in v0.9.33 

func (job *Job) SaveToDisk() (err error)

func (*Job) SetClientgroups added in v0.9.33 

func (job *Job) SetClientgroups(clientgroups string) (err error)

func (*Job) SetDataToken 

func (job *Job) SetDataToken(token string) (err error)

func (*Job) SetExpiration added in v0.9.26 

func (job *Job) SetExpiration(expire string) (err error)

func (*Job) SetFile 

func (job *Job) SetFile(file FormFile) (err error)

func (*Job) SetLastFailed added in v0.9.36 

func (job *Job) SetLastFailed(lastfailed string) (err error)

func (*Job) SetPipeline added in v0.9.26 

func (job *Job) SetPipeline(pipeline string) (err error)

func (*Job) SetPriority added in v0.9.33 

func (job *Job) SetPriority(priority int) (err error)

func (*Job) SetRemainTasks added in v0.9.33 

func (job *Job) SetRemainTasks(remain_tasks int) (err error)

func (*Job) SetState added in v0.9.33 

func (job *Job) SetState(newState string, notes string) (err error)

func (*Job) TaskList 

func (job *Job) TaskList() []*Task

---Task functions

func (*Job) UpdateFile 

func (job *Job) UpdateFile(files FormFiles, field string) (err error)

---Script upload (e.g. field="upload")

func (*Job) UpdateTaskDEPRECATED added in v0.9.33 

func (job *Job) UpdateTaskDEPRECATED(task *Task) (remainTasks int, err error)

invoked to modify job info in mongodb when a task in that job changed to the new status task is already locked

type JobDep added in v0.9.16 

type JobDep struct {
	JobRaw `bson:",inline"`
	Tasks  []*TaskDep `bson:"tasks" json:"tasks"`
}

Deprecated JobDep struct uses deprecated TaskDep struct which uses the deprecated IOmap. Maintained for backwards compatibility. Jobs that cannot be parsed into the Job struct, but can be parsed into the JobDep struct will be translated to the new Job struct. (=deprecated=)

func NewJobDep added in v0.9.33 

func NewJobDep() (job *JobDep)

type JobLog added in v0.9.27 

type JobLog struct {
	Id         string     `bson:"id" json:"id"`
	State      string     `bson:"state" json:"state"`
	UpdateTime time.Time  `bson:"updatetime" json:"updatetime"`
	Notes      string     `bson:"notes" json:"notes"`
	LastFailed string     `bson:"lastfailed" json:"lastfailed"`
	Resumed    int        `bson:"resumed" json:"resumed"`
	Tasks      []*TaskLog `bson:"tasks" json:"tasks"`
}

type JobMap added in v0.9.33 

type JobMap struct {
	RWMutex
	// contains filtered or unexported fields
}

func NewJobMap added in v0.9.33 

func NewJobMap() (t *JobMap)

func (*JobMap) Add added in v0.9.33 

func (jm *JobMap) Add(job *Job) (err error)

func (*JobMap) Get added in v0.9.33 

func (jm *JobMap) Get(jobid string, lock bool) (job *Job, ok bool, err error)

func (*JobMap) Len added in v0.9.33 

func (jm *JobMap) Len() (length int, err error)

type JobMgr 

type JobMgr interface {
	EnqueueTasksByJobId(string) error
	GetActiveJobs() map[string]bool
	IsJobRegistered(string) bool
	GetSuspendJobs() map[string]bool
	SuspendJob(string, string, string) error
	ResumeSuspendedJobByUser(string, *user.User) error
	ResumeSuspendedJobsByUser(*user.User) int
	ResubmitJob(string) error
	DeleteJobByUser(string, *user.User, bool) error
	DeleteSuspendedJobsByUser(*user.User, bool) int
	DeleteZombieJobsByUser(*user.User, bool) int
	RecoverJob(string) error
	RecoverJobs() error
	FinalizeWorkPerf(string, string) error
	SaveStdLog(string, string, string) error
	GetReportMsg(string, string) (string, error)
	RecomputeJob(string, string) error
	UpdateQueueJobInfo(*Job) error
}

type JobMin added in v0.9.8 

type JobMin struct {
	Id            string            `bson:"id" json:"id"`
	Name          string            `bson:"name" json:"name"`
	Size          int64             `bson:"size" json:"size"`
	SubmitTime    time.Time         `bson:"submittime" json:"submittime"`
	CompletedTime time.Time         `bson:"completedtime" json:"completedtime"`
	ComputeTime   int               `bson:"computetime" json:"computetime"`
	Task          []int             `bson:"task" json:"task"`
	State         []string          `bson:"state" json:"state"`
	UserAttr      map[string]string `bson:"userattr" json:"userattr"`
}

type JobPerf 

type JobPerf struct {
	Id     string               `bson:"id" json:"id"`
	Queued int64                `bson:"queued" json:"queued"`
	Start  int64                `bson:"start" json:"start"`
	End    int64                `bson:"end" json:"end"`
	Resp   int64                `bson:"resp" json:"resp"` //End - Queued
	Ptasks map[string]*TaskPerf `bson:"task_stats" json:"task_stats"`
	Pworks map[string]*WorkPerf `bson:"work_stats" json:"work_stats"`
}

func LoadJobPerf 

func LoadJobPerf(id string) (perf *JobPerf, err error)

func NewJobPerf 

func NewJobPerf(id string) *JobPerf

type JobRaw added in v0.9.33 

type JobRaw struct {
	RWMutex
	Id string `bson:"id" json:"id"`
	//Tasks       []*Task   `bson:"tasks" json:"tasks"`
	Acl         acl.Acl   `bson:"acl" json:"-"`
	Info        *Info     `bson:"info" json:"info"`
	Script      script    `bson:"script" json:"-"`
	State       string    `bson:"state" json:"state"`
	Registered  bool      `bson:"registered" json:"registered"`
	RemainTasks int       `bson:"remaintasks" json:"remaintasks"`
	Expiration  time.Time `bson:"expiration" json:"expiration"` // 0 means no expiration
	UpdateTime  time.Time `bson:"updatetime" json:"updatetime"`
	Notes       string    `bson:"notes" json:"notes"`
	LastFailed  string    `bson:"lastfailed" json:"lastfailed"`
	Resumed     int       `bson:"resumed" json:"resumed"`     // number of times the job has been resumed from suspension
	ShockHost   string    `bson:"shockhost" json:"shockhost"` // this is a fall-back default if not specified at a lower level
}

func NewJobRaw added in v0.9.33 

func NewJobRaw() (job *JobRaw)

type JobReaper added in v0.9.26 

type JobReaper struct{}
var (
	Ttl         *JobReaper
	ExpireRegex = regexp.MustCompile(`^(\d+)(M|H|D)$`)
)

func NewJobReaper added in v0.9.26 

func NewJobReaper() *JobReaper

func (*JobReaper) Handle added in v0.9.26 

func (jr *JobReaper) Handle()

type Job_Acl added in v0.9.36 

type Job_Acl struct {
	Acl acl.Acl `bson:"acl" json:"-"`
}

type Job_p added in v0.9.3 

type Job_p struct {
	Tasks []*Task_p `json:"tasks"`
}

type Jobs 

type Jobs []*Job

Job array type

func (*Jobs) GetAll 

func (n *Jobs) GetAll(q bson.M, order string, direction string) (err error)

func (*Jobs) GetAllLimitOffset 

func (n *Jobs) GetAllLimitOffset(q bson.M, limit int, offset int) (err error)

func (*Jobs) GetAllRecent 

func (n *Jobs) GetAllRecent(q bson.M, recent int) (count int, err error)

func (*Jobs) GetAllUnsorted added in v0.9.26 

func (n *Jobs) GetAllUnsorted(q bson.M) (err error)

func (*Jobs) GetJobAt 

func (n *Jobs) GetJobAt(index int) *Job

func (*Jobs) GetPaginated 

func (n *Jobs) GetPaginated(q bson.M, limit int, offset int, order string, direction string) (count int, err error)

func (*Jobs) Init added in v0.9.33 

func (n *Jobs) Init() (err error)

func (*Jobs) Length 

func (n *Jobs) Length() int

func (*Jobs) RLockRecursive added in v0.9.33 

func (n *Jobs) RLockRecursive()

func (*Jobs) RUnlockRecursive added in v0.9.33 

func (n *Jobs) RUnlockRecursive()

type Notice 

type Notice struct {
	WorkId      string
	Status      string
	ClientId    string
	ComputeTime int
	Notes       string
}

type Opts 

type Opts map[string]string

func (*Opts) HasKey 

func (o *Opts) HasKey(key string) bool

func (*Opts) Value 

func (o *Opts) Value(key string) string

type PartInfo 

type PartInfo struct {
	Input         string `bson:"input" json:"input"`
	Index         string `bson:"index" json:"index"`
	TotalIndex    int    `bson:"totalindex" json:"totalindex"`
	MaxPartSizeMB int    `bson:"maxpartsize_mb" json:"maxpartsize_mb"`
	Options       string `bson:"options" json:"-"`
}

type Pipeline 

type Pipeline struct {
	Info  *Info  `bson:"info" json:"info"`
	Tasks []Task `bson:"tasks" json:"tasks"`
}

func NewPipeline 

func NewPipeline() *Pipeline

type ProxyMgr 

type ProxyMgr struct {
	CQMgr
}

func NewProxyMgr 

func NewProxyMgr() *ProxyMgr

func (*ProxyMgr) ClientChecker 

func (qm *ProxyMgr) ClientChecker()

func (*ProxyMgr) ClientHandle added in v0.9.16 

func (qm *ProxyMgr) ClientHandle()

func (*ProxyMgr) DeleteJobByUser added in v0.9.3 

func (qm *ProxyMgr) DeleteJobByUser(jobid string, u *user.User, full bool) (err error)

func (*ProxyMgr) DeleteSuspendedJobsByUser added in v0.9.3 

func (qm *ProxyMgr) DeleteSuspendedJobsByUser(u *user.User, full bool) (num int)

func (*ProxyMgr) DeleteZombieJobsByUser added in v0.9.3 

func (qm *ProxyMgr) DeleteZombieJobsByUser(u *user.User, full bool) (num int)

func (*ProxyMgr) EnqueueTasksByJobId 

func (qm *ProxyMgr) EnqueueTasksByJobId(jobid string) (err error)

func (*ProxyMgr) FetchDataToken 

func (qm *ProxyMgr) FetchDataToken(workid string, clientid string) (token string, err error)

func (*ProxyMgr) FetchPrivateEnv added in v0.9.3 

func (qm *ProxyMgr) FetchPrivateEnv(workid string, clientid string) (env map[string]string, err error)

func (*ProxyMgr) FinalizeWorkPerf 

func (qm *ProxyMgr) FinalizeWorkPerf(string, string) (err error)

func (*ProxyMgr) GetActiveJobs 

func (qm *ProxyMgr) GetActiveJobs() map[string]bool

func (*ProxyMgr) GetJsonStatus added in v0.9.26 

func (qm *ProxyMgr) GetJsonStatus() (status map[string]map[string]int, err error)

func (*ProxyMgr) GetQueue added in v0.9.26 

func (qm *ProxyMgr) GetQueue(name string) interface{}

func (*ProxyMgr) GetReportMsg 

func (qm *ProxyMgr) GetReportMsg(string, string) (report string, err error)

func (*ProxyMgr) GetSuspendJobs 

func (qm *ProxyMgr) GetSuspendJobs() map[string]bool

func (*ProxyMgr) GetTextStatus added in v0.9.26 

func (qm *ProxyMgr) GetTextStatus() string

func (*ProxyMgr) IsJobRegistered 

func (qm *ProxyMgr) IsJobRegistered(id string) bool

func (*ProxyMgr) JobRegister 

func (qm *ProxyMgr) JobRegister() (jid string, err error)

func (*ProxyMgr) Lock added in v0.9.33 

func (qm *ProxyMgr) Lock()

func (*ProxyMgr) NoticeHandle added in v0.9.33 

func (qm *ProxyMgr) NoticeHandle()

func (*ProxyMgr) QueueStatus added in v0.9.23 

func (qm *ProxyMgr) QueueStatus() string

func (*ProxyMgr) RLock added in v0.9.33 

func (qm *ProxyMgr) RLock()

func (*ProxyMgr) RUnlock added in v0.9.33 

func (qm *ProxyMgr) RUnlock()

func (*ProxyMgr) RecomputeJob 

func (qm *ProxyMgr) RecomputeJob(jobid string, stage string) (err error)

recompute jobs from specified task stage

func (*ProxyMgr) RecoverJob added in v0.9.26 

func (qm *ProxyMgr) RecoverJob(id string) (err error)

recover job not in queue

func (*ProxyMgr) RecoverJobs 

func (qm *ProxyMgr) RecoverJobs() (err error)

recover jobs not completed before awe-server restarts

func (*ProxyMgr) RegisterNewClient 

func (qm *ProxyMgr) RegisterNewClient(files FormFiles, cg *ClientGroup) (client *Client, err error)

func (*ProxyMgr) ResubmitJob 

func (qm *ProxyMgr) ResubmitJob(id string) (err error)

re-submit a job in db but not in the queue (caused by server restarting)

func (*ProxyMgr) ResumeQueue added in v0.9.23 

func (qm *ProxyMgr) ResumeQueue()

func (*ProxyMgr) ResumeSuspendedJobByUser added in v0.9.3 

func (qm *ProxyMgr) ResumeSuspendedJobByUser(id string, u *user.User) (err error)

resubmit a suspended job if user has rights

func (*ProxyMgr) ResumeSuspendedJobsByUser added in v0.9.3 

func (qm *ProxyMgr) ResumeSuspendedJobsByUser(u *user.User) (num int)

func (*ProxyMgr) SaveStdLog 

func (qm *ProxyMgr) SaveStdLog(string, string, string) (err error)

func (*ProxyMgr) SuspendJob 

func (qm *ProxyMgr) SuspendJob(jobid string, reason string, id string) (err error)

func (*ProxyMgr) SuspendQueue added in v0.9.23 

func (qm *ProxyMgr) SuspendQueue()

func (*ProxyMgr) TaskHandle added in v0.9.16 

func (qm *ProxyMgr) TaskHandle()

func (*ProxyMgr) Unlock added in v0.9.33 

func (qm *ProxyMgr) Unlock()

func (*ProxyMgr) UpdateQueueJobInfo added in v0.9.33 

func (qm *ProxyMgr) UpdateQueueJobInfo(job *Job) (err error)

func (*ProxyMgr) UpdateQueueLoop added in v0.9.33 

func (qm *ProxyMgr) UpdateQueueLoop()

type RWMutex added in v0.9.33 

type RWMutex struct {
	Name string `bson:"-" json:"-"`
	// contains filtered or unexported fields
}

func (*RWMutex) Init added in v0.9.33 

func (m *RWMutex) Init(name string)

func (*RWMutex) Lock added in v0.9.33 

func (m *RWMutex) Lock() (err error)

func (*RWMutex) LockNamed added in v0.9.33 

func (m *RWMutex) LockNamed(name string) (err error)

func (*RWMutex) RCount added in v0.9.33 

func (m *RWMutex) RCount() (c int)

func (*RWMutex) RList added in v0.9.33 

func (m *RWMutex) RList() (list []string)

func (*RWMutex) RLock added in v0.9.33 

func (m *RWMutex) RLock()

func (*RWMutex) RLockAnon added in v0.9.33 

func (m *RWMutex) RLockAnon()

func (*RWMutex) RLockNamed added in v0.9.33 

func (m *RWMutex) RLockNamed(name string) (rl ReadLock, err error)

func (*RWMutex) RUnlock added in v0.9.33 

func (m *RWMutex) RUnlock()

func (*RWMutex) RUnlockAnon added in v0.9.33 

func (m *RWMutex) RUnlockAnon()

func (*RWMutex) RUnlockNamed added in v0.9.33 

func (m *RWMutex) RUnlockNamed(rl ReadLock)

func (*RWMutex) Unlock added in v0.9.33 

func (m *RWMutex) Unlock()

type ReadLock added in v0.9.33 

type ReadLock struct {
	// contains filtered or unexported fields
}

func (*ReadLock) Get_Id added in v0.9.33 

func (r *ReadLock) Get_Id() string

type ResourceMgr 

type ResourceMgr interface {
	ClientWorkMgr
	JobMgr
	TaskHandle()
	ClientHandle()
	UpdateQueueLoop()
	NoticeHandle()
	GetJsonStatus() (map[string]map[string]int, error)
	GetTextStatus() string
	QueueStatus() string
	GetQueue(string) interface{}
	SuspendQueue()
	ResumeQueue()
	Lock()
	Unlock()
	RLock()
	RUnlock()
}

type ServerMgr 

type ServerMgr struct {
	CQMgr

	//taskLock  sync.RWMutex
	TaskMap TaskMap
	// contains filtered or unexported fields
}

func NewServerMgr 

func NewServerMgr() *ServerMgr

func (*ServerMgr) ClientHandle added in v0.9.16 

func (qm *ServerMgr) ClientHandle()

func (*ServerMgr) CreateAndEnqueueWorkunits added in v0.9.33 

func (qm *ServerMgr) CreateAndEnqueueWorkunits(task *Task) (err error)

func (*ServerMgr) CreateJobPerf 

func (qm *ServerMgr) CreateJobPerf(jobid string)

---perf related methods

func (*ServerMgr) CreateTaskPerf 

func (qm *ServerMgr) CreateTaskPerf(taskid string)

func (*ServerMgr) CreateWorkPerf 

func (qm *ServerMgr) CreateWorkPerf(workid string)

func (*ServerMgr) DeleteJobByUser added in v0.9.3 

func (qm *ServerMgr) DeleteJobByUser(jobid string, u *user.User, full bool) (err error)

func (*ServerMgr) DeleteSuspendedJobsByUser added in v0.9.3 

func (qm *ServerMgr) DeleteSuspendedJobsByUser(u *user.User, full bool) (num int)

func (*ServerMgr) DeleteZombieJobsByUser added in v0.9.3 

func (qm *ServerMgr) DeleteZombieJobsByUser(u *user.User, full bool) (num int)

delete jobs in db with "queued" or "in-progress" state but not in the queue (zombie jobs) that user has access to

func (*ServerMgr) EnqueueTasksByJobId 

func (qm *ServerMgr) EnqueueTasksByJobId(jobid string) (err error)

func (*ServerMgr) FetchDataToken 

func (qm *ServerMgr) FetchDataToken(workid string, clientid string) (token string, err error)

--workunit methds (servermgr implementation)

func (*ServerMgr) FetchPrivateEnv added in v0.9.3 

func (qm *ServerMgr) FetchPrivateEnv(workid string, clientid string) (env map[string]string, err error)

func (*ServerMgr) FetchPrivateEnvs added in v0.9.3 

func (qm *ServerMgr) FetchPrivateEnvs(workid string, clientid string) (envs map[string]string, err error)

func (*ServerMgr) FinalizeJobPerf 

func (qm *ServerMgr) FinalizeJobPerf(jobid string)

func (*ServerMgr) FinalizeTaskPerf 

func (qm *ServerMgr) FinalizeTaskPerf(task *Task)

func (*ServerMgr) FinalizeWorkPerf 

func (qm *ServerMgr) FinalizeWorkPerf(workid string, reportfile string) (err error)

func (*ServerMgr) GetActiveJobs 

func (qm *ServerMgr) GetActiveJobs() (ajobs map[string]bool)

func (*ServerMgr) GetJsonStatus added in v0.9.26 

func (qm *ServerMgr) GetJsonStatus() (status map[string]map[string]int, err error)

func (*ServerMgr) GetQueue added in v0.9.26 

func (qm *ServerMgr) GetQueue(name string) interface{}

func (*ServerMgr) GetReportMsg 

func (qm *ServerMgr) GetReportMsg(workid string, logname string) (report string, err error)

func (*ServerMgr) GetSuspendJobs 

func (qm *ServerMgr) GetSuspendJobs() (sjobs map[string]bool)

func (*ServerMgr) GetTextStatus added in v0.9.26 

func (qm *ServerMgr) GetTextStatus() string

func (*ServerMgr) IsJobRegistered 

func (qm *ServerMgr) IsJobRegistered(id string) bool

func (*ServerMgr) Lock added in v0.9.33 

func (qm *ServerMgr) Lock()

func (*ServerMgr) LogJobPerf 

func (qm *ServerMgr) LogJobPerf(jobid string)

func (*ServerMgr) NoticeHandle added in v0.9.33 

func (qm *ServerMgr) NoticeHandle()

func (*ServerMgr) QueueStatus added in v0.9.23 

func (qm *ServerMgr) QueueStatus() string

func (*ServerMgr) RLock added in v0.9.33 

func (qm *ServerMgr) RLock()

func (*ServerMgr) RUnlock added in v0.9.33 

func (qm *ServerMgr) RUnlock()

func (*ServerMgr) RecomputeJob 

func (qm *ServerMgr) RecomputeJob(jobid string, stage string) (err error)

recompute job from specified task stage

func (*ServerMgr) RecoverJob added in v0.9.26 

func (qm *ServerMgr) RecoverJob(id string) (err error)

recover a job in db that is missing from queue (caused by server restarting)

func (*ServerMgr) RecoverJobs 

func (qm *ServerMgr) RecoverJobs() (err error)

recover jobs not completed before awe-server restarts

func (*ServerMgr) ResubmitJob 

func (qm *ServerMgr) ResubmitJob(jobid string) (err error)

recompute job from beginning

func (*ServerMgr) ResumeQueue added in v0.9.23 

func (qm *ServerMgr) ResumeQueue()

func (*ServerMgr) ResumeSuspendedJobByUser added in v0.9.3 

func (qm *ServerMgr) ResumeSuspendedJobByUser(id string, u *user.User) (err error)

resubmit a suspended job if the user is authorized

func (*ServerMgr) ResumeSuspendedJobsByUser added in v0.9.3 

func (qm *ServerMgr) ResumeSuspendedJobsByUser(u *user.User) (num int)

func (*ServerMgr) SaveStdLog 

func (qm *ServerMgr) SaveStdLog(workid string, logname string, tmppath string) (err error)

func (*ServerMgr) ShowTasks 

func (qm *ServerMgr) ShowTasks()

show functions used in debug

func (*ServerMgr) SuspendJob 

func (qm *ServerMgr) SuspendJob(jobid string, reason string, id string) (err error)

func (*ServerMgr) SuspendQueue added in v0.9.23 

func (qm *ServerMgr) SuspendQueue()

func (*ServerMgr) TaskHandle added in v0.9.16 

func (qm *ServerMgr) TaskHandle()

func (*ServerMgr) Unlock added in v0.9.33 

func (qm *ServerMgr) Unlock()

func (*ServerMgr) UpdateJobPerfStartTime 

func (qm *ServerMgr) UpdateJobPerfStartTime(jobid string)

func (*ServerMgr) UpdateJobTaskToInProgress 

func (qm *ServerMgr) UpdateJobTaskToInProgress(works []*Workunit)

update job/task states from "queued" to "in-progress" once the first workunit is checked out

func (*ServerMgr) UpdateQueueJobInfo added in v0.9.33 

func (qm *ServerMgr) UpdateQueueJobInfo(job *Job) (err error)

update job info for in-memory data structures

func (*ServerMgr) UpdateQueueLoop added in v0.9.33 

func (qm *ServerMgr) UpdateQueueLoop()

func (*ServerMgr) UpdateTaskPerfStartTime 

func (qm *ServerMgr) UpdateTaskPerfStartTime(taskid string)

type StandardResponse added in v0.9.33 

type StandardResponse struct {
	S int         `json:"status"`
	D interface{} `json:"data"`
	E []string    `json:"error"`
}

func NotifyWorkunitProcessedWithLogs 

func NotifyWorkunitProcessedWithLogs(work *Workunit, perf *WorkPerf, sendstdlogs bool) (response *StandardResponse, err error)

type StringLocked added in v0.9.33 

type StringLocked struct {
	sync.Mutex
	// contains filtered or unexported fields
}

func (*StringLocked) Get added in v0.9.33 

func (s *StringLocked) Get() string

func (*StringLocked) Set added in v0.9.33 

func (s *StringLocked) Set(value string)

type Task 

type Task struct {
	TaskRaw `bson:",inline"`
	Inputs  []*IO `bson:"inputs" json:"inputs"`
	Outputs []*IO `bson:"outputs" json:"outputs"`
	Predata []*IO `bson:"predata" json:"predata"`
}

func NewTask 

func NewTask(job *Job, task_id string) (t *Task, err error)

func (*Task) CreateIndex added in v0.9.4 

func (task *Task) CreateIndex() (err error)

checks and creates indices on shock node if needed

func (*Task) CreateWorkunits added in v0.9.33 

func (task *Task) CreateWorkunits() (wus []*Workunit, err error)

func (*Task) DeleteInput added in v0.9.12 

func (task *Task) DeleteInput() (modified int)

func (*Task) DeleteOutput added in v0.9.3 

func (task *Task) DeleteOutput() (modified int)

func (*Task) GetOutput added in v0.9.36 

func (task *Task) GetOutput(filename string) (output *IO, err error)

func (*Task) GetOutputs added in v0.9.33 

func (task *Task) GetOutputs() (outputs []*IO, err error)

func (*Task) GetTaskLogs added in v0.9.27 

func (task *Task) GetTaskLogs() (tlog *TaskLog)

func (*Task) IncrementComputeTime added in v0.9.33 

func (task *Task) IncrementComputeTime(inc_time int) (err error)

func (*Task) IncrementRemainWork added in v0.9.33 

func (task *Task) IncrementRemainWork(inc int, writelock bool) (remainwork int, err error)

func (*Task) Init added in v0.9.36 

func (task *Task) Init(job *Job) (changed bool, err error)

func (*Task) InitPartIndex 

func (task *Task) InitPartIndex() (err error)

get part size based on partition/index info if fail to get index info, task.TotalWork fall back to 1 and return nil

func (*Task) SetRemainWork added in v0.9.33 

func (task *Task) SetRemainWork(num int, writelock bool) (err error)

func (*Task) UpdateInputs added in v0.9.36 

func (task *Task) UpdateInputs() (err error)

func (*Task) UpdateOutputs added in v0.9.36 

func (task *Task) UpdateOutputs() (err error)

func (*Task) UpdatePredata added in v0.9.36 

func (task *Task) UpdatePredata() (err error)

func (*Task) UpdateState_DEPRECATED added in v0.9.36 

func (task *Task) UpdateState_DEPRECATED(newState string) string

type TaskDep added in v0.9.16 

type TaskDep struct {
	TaskRaw `bson:",inline"`
	Inputs  IOmap `bson:"inputs" json:"inputs"`
	Outputs IOmap `bson:"outputs" json:"outputs"`
	Predata IOmap `bson:"predata" json:"predata"`
}

Deprecated JobDep struct uses deprecated TaskDep struct which uses the deprecated IOmap. Maintained for backwards compatibility. Jobs that cannot be parsed into the Job struct, but can be parsed into the JobDep struct will be translated to the new Job struct. (=deprecated=)

type TaskLog added in v0.9.27 

type TaskLog struct {
	Id            string     `bson:"taskid" json:"taskid"`
	State         string     `bson:"state" json:"state"`
	TotalWork     int        `bson:"totalwork" json:"totalwork"`
	CompletedDate time.Time  `bson:"completedDate" json:"completeddate"`
	Workunits     []*WorkLog `bson:"workunits" json:"workunits"`
}

type TaskMap added in v0.9.33 

type TaskMap struct {
	RWMutex
	// contains filtered or unexported fields
}

func NewTaskMap added in v0.9.33 

func NewTaskMap() (t *TaskMap)

func (*TaskMap) Add added in v0.9.33 

func (tm *TaskMap) Add(task *Task)

func (*TaskMap) Delete added in v0.9.33 

func (tm *TaskMap) Delete(taskid string) (task *Task, ok bool)

func (*TaskMap) Get added in v0.9.33 

func (tm *TaskMap) Get(taskid string, lock bool) (task *Task, ok bool, err error)

func (*TaskMap) GetTasks added in v0.9.33 

func (tm *TaskMap) GetTasks() (tasks []*Task, err error)

func (*TaskMap) Len added in v0.9.33 

func (tm *TaskMap) Len() (length int, err error)

func (*TaskMap) SetStateDEPRECATED added in v0.9.33 

func (tm *TaskMap) SetStateDEPRECATED(id string, new_state string) (err error)

TODO remove ?

type TaskPerf 

type TaskPerf struct {
	Queued       int64   `bson:"queued" json:"queued"`
	Start        int64   `bson:"start" json:"start"`
	End          int64   `bson:"end" json:"end"`
	Resp         int64   `bson:"resp" json:"resp"` //End -Queued
	InFileSizes  []int64 `bson:"size_infile" json:"size_infile"`
	OutFileSizes []int64 `bson:"size_outfile" json:"size_outfile"`
}

func NewTaskPerf 

func NewTaskPerf(id string) *TaskPerf

type TaskRaw added in v0.9.33 

type TaskRaw struct {
	RWMutex
	Id          string    `bson:"taskid" json:"taskid"`
	JobId       string    `bson:"jobid" json:"jobid"`
	Info        *Info     `bson:"-" json:"-"`
	Cmd         *Command  `bson:"cmd" json:"cmd"`
	Partition   *PartInfo `bson:"partinfo" json:"-"`
	DependsOn   []string  `bson:"dependsOn" json:"dependsOn"` // only needed if dependency cannot be inferred from Input.Origin
	TotalWork   int       `bson:"totalwork" json:"totalwork"`
	MaxWorkSize int       `bson:"maxworksize"   json:"maxworksize"`
	RemainWork  int       `bson:"remainwork" json:"remainwork"`
	//WorkStatus  []string  `bson:"workstatus" json:"-"`
	State string `bson:"state" json:"state"`
	//Skip          int               `bson:"skip" json:"-"`
	CreatedDate   time.Time         `bson:"createdDate" json:"createddate"`
	StartedDate   time.Time         `bson:"startedDate" json:"starteddate"`
	CompletedDate time.Time         `bson:"completedDate" json:"completeddate"`
	ComputeTime   int               `bson:"computetime" json:"computetime"`
	UserAttr      map[string]string `bson:"userattr" json:"userattr"`
	ClientGroups  string            `bson:"clientgroups" json:"clientgroups"`
}

func NewTaskRaw added in v0.9.33 

func NewTaskRaw(task_id string, info *Info) TaskRaw

func (*TaskRaw) GetDependsOn added in v0.9.33 

func (task *TaskRaw) GetDependsOn() (dep []string, err error)

func (*TaskRaw) GetId added in v0.9.33 

func (task *TaskRaw) GetId() (id string, err error)

func (*TaskRaw) GetJobId added in v0.9.33 

func (task *TaskRaw) GetJobId() (id string, err error)

func (*TaskRaw) GetState added in v0.9.33 

func (task *TaskRaw) GetState() (state string, err error)

func (*TaskRaw) GetStateNamed added in v0.9.33 

func (task *TaskRaw) GetStateNamed(name string) (state string, err error)

only for debugging purposes

func (*TaskRaw) InitRaw added in v0.9.36 

func (task *TaskRaw) InitRaw(job *Job) (changed bool, err error)

func (*TaskRaw) SetCompletedDate_DEPRECATED added in v0.9.33 

func (task *TaskRaw) SetCompletedDate_DEPRECATED(date time.Time) (err error)

func (*TaskRaw) SetCreatedDate added in v0.9.33 

func (task *TaskRaw) SetCreatedDate(t time.Time) (err error)

func (*TaskRaw) SetStartedDate added in v0.9.33 

func (task *TaskRaw) SetStartedDate(t time.Time) (err error)

func (*TaskRaw) SetState added in v0.9.33 

func (task *TaskRaw) SetState(new_state string) (err error)

type Task_p added in v0.9.3 

type Task_p struct {
	Cmd *Command_p `json:"cmd"`
}

type WorkList 

type WorkList []*Workunit

queuing/prioritizing related functions

func (WorkList) Len 

func (wl WorkList) Len() int

func (WorkList) Swap 

func (wl WorkList) Swap(i, j int)

type WorkLog added in v0.9.27 

type WorkLog struct {
	Id   string            `bson:"wuid" json:"wuid"`
	Rank int               `bson:"rank" json:"rank"`
	Logs map[string]string `bson:"logs" json:"logs"`
}

func NewWorkLog added in v0.9.27 

func NewWorkLog(tid string, rank int) (wlog *WorkLog)

type WorkMgr 

type WorkMgr interface {
	GetWorkById(string) (*Workunit, error)
	ShowWorkunits(string) ([]*Workunit, error)
	ShowWorkunitsByUser(string, *user.User) []*Workunit
	CheckoutWorkunits(string, string, *Client, int64, int) ([]*Workunit, error)
	NotifyWorkStatus(Notice)
	EnqueueWorkunit(*Workunit) error
	FetchDataToken(string, string) (string, error)
	FetchPrivateEnv(string, string) (map[string]string, error)
}

type WorkPerf 

type WorkPerf struct {
	Queued             int64   `bson:"queued" json:"queued"`                   // WQ (queued at server or client, depending on who creates it)
	Done               int64   `bson:"done" json:"done"`                       // WD (done at server)
	Resp               int64   `bson:"resp" json:"resp"`                       // Done - Queued (server metric)
	Checkout           int64   `bson:"checkout" json:"checkout"`               // checkout at client
	Deliver            int64   `bson:"deliver" json:"deliver"`                 // done at client
	ClientResp         int64   `bson:"clientresp" json:"clientresp"`           // Deliver - Checkout (client metric)
	PreDataIn          float64 `bson:"time_predata_in" json:"time_predata_in"` // time in seconds for downloading prerequisite data at client
	DataIn             float64 `bson:"time_data_in" json:"time_data_in"`       // time in seconds for input data move-in at client
	DataOut            float64 `bson:"time_data_out" json:"time_data_out"`     // time in seconds for output data move-out at client
	Runtime            int64   `bson:"runtime" json:"runtime"`                 // time in seconds for computation at client
	DockerPrep         int64   `bson:"dockerprep" json:"dockerprep"`           // time in seconds for docker preparation on client
	MaxMemUsage        int64   `bson:"max_mem_usage" json:"max_mem_usage"`     // maxium memory consumption
	MaxMemoryTotalRss  int64   `bson:"max_memory_total_rss" json:"max_memory_total_rss"`
	MaxMemoryTotalSwap int64   `bson:"max_memory_total_swap" json:"max_memory_total_swap"`
	ClientId           string  `bson:"client_id" json:"client_id"`
	PreDataSize        int64   `bson:"size_predata" json:"size_predata"` //predata moved over network
	InFileSize         int64   `bson:"size_infile" json:"size_infile"`   //input file moved over network
	OutFileSize        int64   `bson:"size_outfile" json:"size_outfile"` //outpuf file moved over network
}

func NewWorkPerf 

func NewWorkPerf(id string) *WorkPerf

type WorkQueue added in v0.9.33 

type WorkQueue struct {
	Queue    WorkunitMap // WORK_STAT_QUEUED - waiting workunits
	Checkout WorkunitMap // WORK_STAT_CHECKOUT - workunits being checked out
	Suspend  WorkunitMap // WORK_STAT_SUSPEND - suspended workunits
	// contains filtered or unexported fields
}

func NewWorkQueue added in v0.9.33 

func NewWorkQueue() *WorkQueue

func (*WorkQueue) Add added in v0.9.33 

func (wq *WorkQueue) Add(workunit *Workunit) (err error)

func (*WorkQueue) Clean added in v0.9.33 

func (wq *WorkQueue) Clean() (workids []string)

func (*WorkQueue) Delete added in v0.9.33 

func (wq *WorkQueue) Delete(id string) (err error)

func (*WorkQueue) Get added in v0.9.33 

func (wq *WorkQueue) Get(id string) (w *Workunit, ok bool, err error)

func (*WorkQueue) GetAll added in v0.9.33 

func (wq *WorkQueue) GetAll() (worklist []*Workunit, err error)

func (*WorkQueue) GetForJob added in v0.9.33 

func (wq *WorkQueue) GetForJob(jobid string) (worklist []*Workunit, err error)

func (*WorkQueue) Has added in v0.9.33 

func (wq *WorkQueue) Has(id string) (has bool, err error)

func (*WorkQueue) Len added in v0.9.33 

func (wq *WorkQueue) Len() (int, error)

func (*WorkQueue) StatusChange added in v0.9.33 

func (wq *WorkQueue) StatusChange(id string, workunit *Workunit, new_status string) (err error)

type Workflow 

type Workflow struct {
	WfInfo     awf_info          `bson:"workflow_info" json:"workflow_info"`
	JobInfo    awf_jobinfo       `bson:"job_info" json:"job_info"`
	RawInputs  map[string]string `bson:"raw_inputs" json:"raw_inputs"`
	Variables  map[string]string `bson:"variables" json:"variables"`
	DataServer string            `bson:"data_server" json:"data_server"`
	Tasks      []*awf_task       `bson:"tasks" json:"tasks"`
}

type WorkflowMgr 

type WorkflowMgr struct {
	// contains filtered or unexported fields
}
var (
	AwfMgr *WorkflowMgr
)

func NewWorkflowMgr 

func NewWorkflowMgr() *WorkflowMgr

func (*WorkflowMgr) AddWorkflow 

func (wfm *WorkflowMgr) AddWorkflow(name string, awf *Workflow)

func (*WorkflowMgr) GetAllWorkflows 

func (wfm *WorkflowMgr) GetAllWorkflows() (workflows []*Workflow)

func (*WorkflowMgr) GetWorkflow 

func (wfm *WorkflowMgr) GetWorkflow(name string) (awf *Workflow, err error)

func (*WorkflowMgr) LoadWorkflows 

func (wfm *WorkflowMgr) LoadWorkflows() (err error)

type Workunit 

type Workunit struct {
	Id      string   `bson:"wuid" json:"wuid"`
	Info    *Info    `bson:"info" json:"info"`
	Inputs  []*IO    `bson:"inputs" json:"inputs"`
	Outputs []*IO    `bson:"outputs" json:"outputs"`
	Predata []*IO    `bson:"predata" json:"predata"`
	Cmd     *Command `bson:"cmd" json:"cmd"`
	//	App          *App              `bson:"app" json:"app"`
	Rank         int               `bson:"rank" json:"rank"`
	TotalWork    int               `bson:"totalwork" json:"totalwork"`
	Partition    *PartInfo         `bson:"part" json:"part"`
	State        string            `bson:"state" json:"state"`
	Failed       int               `bson:"failed" json:"failed"`
	CheckoutTime time.Time         `bson:"checkout_time" json:"checkout_time"`
	Client       string            `bson:"client" json:"client"`
	ComputeTime  int               `bson:"computetime" json:"computetime"`
	Notes        string            `bson:"notes" json:"notes"`
	UserAttr     map[string]string `bson:"userattr" json:"userattr"`
}

func NewWorkunit 

func NewWorkunit(task *Task, rank int) *Workunit

func (*Workunit) CDworkpath 

func (work *Workunit) CDworkpath() (err error)

func (*Workunit) IndexType 

func (work *Workunit) IndexType() (indextype string)

func (*Workunit) Mkdir 

func (work *Workunit) Mkdir() (err error)

func (*Workunit) Part 

func (work *Workunit) Part() (part string)

calculate the range of data part algorithm: try to evenly distribute indexed parts to workunits e.g. totalWork=4, totalParts=10, then each workunits have parts 3,3,2,2

func (*Workunit) Path 

func (work *Workunit) Path() string

func (*Workunit) RemoveDir 

func (work *Workunit) RemoveDir() (err error)

func (*Workunit) SetState added in v0.9.36 

func (work *Workunit) SetState(new_state string)

type WorkunitMap added in v0.9.33 

type WorkunitMap struct {
	RWMutex
	Map map[string]*Workunit
}

func NewWorkunitMap added in v0.9.33 

func NewWorkunitMap() *WorkunitMap

func (*WorkunitMap) Delete added in v0.9.33 

func (wm *WorkunitMap) Delete(id string) (err error)

func (*WorkunitMap) Get added in v0.9.33 

func (wm *WorkunitMap) Get(id string) (workunit *Workunit, ok bool, err error)

func (*WorkunitMap) GetWorkunits added in v0.9.33 

func (wm *WorkunitMap) GetWorkunits() (workunits []*Workunit, err error)

func (*WorkunitMap) Len added in v0.9.33 

func (wm *WorkunitMap) Len() (length int, err error)

func (*WorkunitMap) Set added in v0.9.33 

func (wm *WorkunitMap) Set(workunit *Workunit) (err error)

type WorkunitsSortby added in v0.9.8 

type WorkunitsSortby struct {
	Order     string
	Direction string
	Workunits []*Workunit
}

func (WorkunitsSortby) Len added in v0.9.8 

func (w WorkunitsSortby) Len() int

func (WorkunitsSortby) Less added in v0.9.8 

func (w WorkunitsSortby) Less(i, j int) bool

func (WorkunitsSortby) Swap added in v0.9.8 

func (w WorkunitsSortby) Swap(i, j int)

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.