README
¶
gocra
gocra is a port to Cipres Restful API (CRA) using golang.
When using gocra alway consut the CRA User Guide along with gocra documentation.
gocra is capable of submmiting jobs, monitoring jobs status, download results and delete jobs.
You need a cra_auth.json file (the source code contais an example that can be edited) in your user home folder containing you credentials used to login to Cipres API.
{
"url": "https://cipresrest.sdsc.edu/cipresrest/v1",
"appid": "Your application ID",
"username": "Your Username",
"password": "Your Password"
}
Here is an example of gocran package usage:
package main
import (
"fmt"
"log"
"os"
"time"
"github.com/Godrigos/gocra"
)
func main() {
var jr gocra.JobResults
var total float64
jl := gocra.ListJobs()
count := len(jl.Jobs.Jobstatus)
// Ends the execution if there is no job on server.
if count == 0 {
fmt.Println("There are no jobs on Cipres servers.")
os.Exit(0)
}
for i := range jl.Jobs.Jobstatus {
job := gocra.JobStat(jl.Jobs.Jobstatus[i].SelfURI.URL)
wdir := gocra.WorkDir(job)
jr = gocra.JobResult(job)
// Destination folder for the downloaded files
destDir := "/home/user/Downloads"
/* Based on job state passed by jobStage field from a JobStatus structure
prints to stdout the actual job stage and download its results if the
job is completed. */
switch {
case job.JobStage == "QUEUE":
fmt.Printf("The job \033[41;1;37m%s\033[0m has been validated"+
" and placed in CIPRES's queue.\n", job.Metadata.Entry.Value)
fmt.Printf(" \u25CF Job Handle: %s.\n", job.JobHandle)
case job.JobStage == "COMMANDRENDERING":
fmt.Printf("The job \033[41;1;37m%s\033[0m has reached the head"+
" of the queue and CIPRES has created the command line that"+
" will be run.\n", job.Metadata.Entry.Value)
fmt.Printf(" \u25CF Job Handle: %s.\n", job.JobHandle)
case job.JobStage == "INPUTSTAGING":
fmt.Printf("CIPRES has created a temporary working directory"+
" for the job \033[41;1;37m%s\033[0m on the execution host"+
" and copied the input files over.\n",
job.Metadata.Entry.Value)
fmt.Printf(" \u25CF Job Handle: %s.\n", job.JobHandle)
case job.JobStage == "SUBMITTED":
fmt.Printf("The job \033[41;1;37m%s\033[0m has been submited"+
" to the scheduler on the execution host.\n",
job.Metadata.Entry.Value)
fmt.Printf(" \u25CF Job Handle: %s.\n", job.JobHandle)
fmt.Println(" \u25cf Working Directory Files List:")
for j := range wdir.Jobfiles.Jobfile {
fmt.Printf(" %-25s %12.2f kB\n",
wdir.Jobfiles.Jobfile[j].Filename,
float64(wdir.Jobfiles.Jobfile[j].Length)/1000)
total += float64(wdir.Jobfiles.Jobfile[j].Length) / 1000
}
fmt.Printf(" \u25CF Total size %29.2f kB\n\n", total)
case job.JobStage == "LOAD_RESULTS":
fmt.Printf("The job \033[41;1;37m%s\033[0m has finished running"+
" on the execution host and CIPRES has begun to transfer the"+
" results.\n", job.Metadata.Entry.Value)
fmt.Printf(" \u25CF Job Handle: %s.\n", job.JobHandle)
case job.JobStage == "COMPLETED":
fmt.Printf("Job \033[41;1;37m%s\033[0m results successfully"+
" transferred and available.\n", job.Metadata.Entry.Value)
fmt.Printf(" \u25CF Job Handle: %s.\n", job.JobHandle)
fmt.Println(" \u25cf Results Files List:")
for j := range jr.Jobfiles.Jobfile {
fmt.Printf(" %-25s %12.2f kB\n",
jr.Jobfiles.Jobfile[j].Filename,
float64(jr.Jobfiles.Jobfile[j].Length)/1000)
total += float64(jr.Jobfiles.Jobfile[j].Length) / 1000
}
fmt.Printf("\nDownloading files of job %s (%.2f MB)...\n",
job.Metadata.Entry.Value, total/1000)
// Parallel downloading
gocra.DownloadAll(jr, job, destDir)
gocra.Delete(job.SelfURI.URL)
default:
fmt.Println("Can not define job state. Try again later!")
}
/* When there is more than one job on CIPRES server, applies a delay
between jobs access, acconding to the MinPollIntervalSeconds
field from a JobStatus structure */
if count > 1 {
time.Sleep(time.Duration(job.MinPollIntervalSeconds) * time.Second)
count--
}
}
}
Documentation
¶
Index ¶
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
func Delete ¶
func Delete(url string)
Delete send a request to delete a completed job and its corresponding files. Must be only executes after the results have been already downloaded.
func DownloadAll ¶
func DownloadAll(jr JobResults, job JobStatus, destDir string)
DownloadAll will use parallelization to get all the result files from Cipres server. It will create a new folder at the given destination directory, named as the job metadata clientJobName or the jobhandle code if the previuos field is empty.
func DownloadFile ¶
DownloadFile will download a url to a local file. It's efficient because it will write as it downloads and not load the whole file into memory. The target folder should be given as the first funtion argument, followed by the file url. Returns an error if something goes wrong.
func ParseConfig ¶
ParseConfig can extract Visible Parameters and Metadata from a plain text file and return a map ready to use for submission. It is usefull to use with data copied from Cipres Tool Configuration Helper (https://www.phylo.org/restusers/docs/cipresXml) output and saved to a text file.
The data should be in the following format: tool = IQTREE_XSEDE, with each parameter or metadata in a different line.
Infile Parameters should be entered separately as the file parameter of submit function, once they are processed differently by the function.
Types ¶
type Auth ¶
type Auth struct {
URL string `json:"url"`
AppID string `json:"appid"`
User string `json:"username"`
Passwd string `json:"password"`
}
Auth stores the authentication data imported from cra_auth.json
type JobResults ¶
type JobResults struct {
XMLName xml.Name `xml:"results"`
Text string `xml:",chardata"`
Jobfiles struct {
Text string `xml:",chardata"`
Jobfile []struct {
Text string `xml:",chardata"`
DownloadURI struct {
Text string `xml:",chardata"`
URL string `xml:"url"`
Rel string `xml:"rel"`
Title string `xml:"title"`
} `xml:"downloadUri"`
JobHandle string `xml:"jobHandle"`
Filename string `xml:"filename"`
Length int `xml:"length"`
ParameterName string `xml:"parameterName"`
OutputDocumentID string `xml:"outputDocumentId"`
} `xml:"jobfile"`
} `xml:"jobfiles"`
}
JobResults stores a finished job result information retrieved from CIPRES server
func JobResult ¶
func JobResult(jb JobStatus) JobResults
JobResult GETs a finished job files list from CIPRES server based on a job handle provided by jobStatus function and returns a JobResults structure
type JobStatus ¶
type JobStatus struct {
XMLName xml.Name `xml:"jobstatus"`
Text string `xml:",chardata"`
SelfURI struct {
Text string `xml:",chardata"`
URL string `xml:"url"`
Rel string `xml:"rel"`
Title string `xml:"title"`
} `xml:"selfUri"`
JobHandle string `xml:"jobHandle"`
JobStage string `xml:"jobStage"`
TerminalStage bool `xml:"terminalStage"`
Failed string `xml:"failed"`
Metadata struct {
Text string `xml:",chardata"`
Entry struct {
Text string `xml:",chardata"`
Key string `xml:"key"`
Value string `xml:"value"`
} `xml:"entry"`
} `xml:"metadata"`
DateSubmitted string `xml:"dateSubmitted"`
ResultsURI struct {
Text string `xml:",chardata"`
URL string `xml:"url"`
Rel string `xml:"rel"`
Title string `xml:"title"`
} `xml:"resultsUri"`
WorkingDirURI struct {
Text string `xml:",chardata"`
URL string `xml:"url"`
Rel string `xml:"rel"`
Title string `xml:"title"`
} `xml:"workingDirUri"`
Messages struct {
Text string `xml:",chardata"`
Message struct {
Chardata string `xml:",chardata"`
Timestamp string `xml:"timestamp"`
Stage string `xml:"stage"`
Text string `xml:"text"`
} `xml:"message"`
} `xml:"messages"`
MinPollIntervalSeconds int `xml:"minPollIntervalSeconds"`
}
JobStatus stores a job status information retrieved from CIPRES server
func JobStat ¶
JobStat GETs a job handle url (as s string) information from CIPRES server provided by listJobs function and returns a JobStatus structure
func Submit ¶
func Submit(params map[string]string, path map[string]string, validate bool) (JobStatus, *bytes.Buffer)
Submit a job and returns a JobStatus object and pointer to a bytes.Buffer.
params is a map where each key is a Visible parameter or metadata and the value is the string to assign to the parameter.
For example: {
"vparam.toolId" : "CLUSTALW",
"vparam.runtime_" : "1",
}
path is a map where each key is an InFile parameter and the value is the full path of the file.
For example: {
"input.infile_" : "/samplefiles/sample1_in.fasta,
"input.usetree_" : "/samplefiles/guidetree.dnd",
}
See https://www.phylo.org/restusers/docs/guide.html#ConfigureParams for more info on files, parameters and metadata accepted by each tool.
If you want to validate you submission before actually doing it, set validate to true.
type JobsList ¶
type JobsList struct {
XMLName xml.Name `xml:"joblist"`
Text string `xml:",chardata"`
Title string `xml:"title"`
Jobs struct {
Text string `xml:",chardata"`
Jobstatus []struct {
Text string `xml:",chardata"`
SelfURI struct {
Text string `xml:",chardata"`
URL string `xml:"url"`
Rel string `xml:"rel"`
Title string `xml:"title"`
} `xml:"selfUri"`
} `xml:"jobstatus"`
} `xml:"jobs"`
}
JobsList stores jobs list information retrieved from CIPRES server
type WorkingDir ¶
type WorkingDir struct {
XMLName xml.Name `xml:"workingdir"`
Text string `xml:",chardata"`
Jobfiles struct {
Text string `xml:",chardata"`
Jobfile []struct {
Text string `xml:",chardata"`
DownloadURI struct {
Text string `xml:",chardata"`
URL string `xml:"url"`
Rel string `xml:"rel"`
Title string `xml:"title"`
} `xml:"downloadUri"`
Filename string `xml:"filename"`
Length int `xml:"length"`
DateModified string `xml:"dateModified"`
ParameterName string `xml:"parameterName"`
OutputDocumentID string `xml:"outputDocumentId"`
} `xml:"jobfile"`
} `xml:"jobfiles"`
}
WorkingDir stores a running job files list retrieved from CIPRES server
func WorkDir ¶
func WorkDir(job JobStatus) WorkingDir
WorkDir GETs a job working directory files list from CIPRES server based on a job handle provided by jobStatus function and returns a WorkingDir structure
Source Files