README
¶
Google Client Api Batch Utility in Go
The batch package implements a batch service that processes client api calls from various client api go packages and send the queued calls as a single batch call. Individual responses may then be interrogated for errors and the responses processed. For specifics of the batch protocol see https://cloud.google.com/storage/docs/json_api/v1/how-tos/batch
Examples may be found in the example_test.go file.
Documentation
¶
Overview ¶
Batch implements a service to use the google client api batch protocol. For specifics of the batch protocol see https://cloud.google.com/storage/docs/json_api/v1/how-tos/batch
Index ¶
Examples ¶
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
This section is empty.
Types ¶
type Credentialer ¶
Credentialer returns an oauth2 access token.
var ServiceCredentials Credentialer = nil
ServiceCredentials is used to indicate that the batch should use the Service client to authorize the call.
type Oauth2Credentials ¶
type Oauth2Credentials struct {
oauth2.TokenSource
}
Oauth2Credentials wraps an oauth2.TokenSource to make it a Credentialer.
func (*Oauth2Credentials) Authorization ¶
func (o *Oauth2Credentials) Authorization() (string, error)
Authorization returns the authorization string from the TokenSource.
type Result ¶
type Result struct {
// Value is the json decode response
Value interface{}
// Tag is copied from corresponding Request.tag and is used to pass data
// for further processing of the result.
Tag interface{}
// Err is the error result from a call
Err error
}
Results contains data returned by an individual call.
type Service ¶
type Service struct {
// Is set to http.DefaultClient if nil. An oauth2.Client may be used
// to authorize the requests or each individual request may have its
// own credential removing the need for an authorizing client.
Client *http.Client
// MaxCalls is the maximum number of calls that may be sent in a batch.
// The value must be between 1 and 1000. If zero, 1000 is assumed.
MaxCalls int
// contains filtered or unexported fields
}
Service collects the batch calls via its Queue method and executes them via Do().
Example (Calendar) ¶
package main
import (
"github.com/jfcote87/api/googleapi"
"github.com/jfcote87/batch"
"log"
"net/http"
cal "google.golang.org/api/calendar/v3"
)
type DatabaseEvent struct {
Id string `json:"id"`
Title string `json:"nm"`
Start string `json:"start"`
End string `json:"end"`
Description string `json:"desc"`
Link string `json:"link"`
Loc string `json:"loc"`
}
func main() {
var calendarId string = "xxxxxxxxxxxxx@group.calendar.google.com"
var dbEvents []DatabaseEvent = getEventData()
var oauthClient *http.Client = getOauthClient()
// Read through slice of EventFromDb and add to batch. Then call Do() to send
// and process responses
bsv := batch.Service{Client: oauthClient}
calsv, _ := cal.New(googleapi.BatchClient())
for _, dbEvent := range dbEvents {
// create Event
event := &cal.Event{
Summary: dbEvent.Title,
Description: dbEvent.Description,
Location: dbEvent.Loc,
Start: &cal.EventDateTime{DateTime: dbEvent.Start},
End: &cal.EventDateTime{DateTime: dbEvent.End},
Reminders: &cal.EventReminders{UseDefault: false},
Transparency: "transparent",
Source: &cal.EventSource{Title: "Web Link", Url: dbEvent.Link},
ExtendedProperties: &cal.EventExtendedProperties{Shared: map[string]string{"DBID": dbEvent.Id}},
}
event, err := calsv.Events.Insert(calendarId, event).Do()
// queue new request in batch service
if err = bsv.Queue(err, dbEvent, batch.ServiceCredentials); err != nil {
log.Println(err)
return
}
}
// execute batch
results, err := bsv.Do()
if err != nil {
log.Println(err)
return
}
for _, r := range results {
var event *cal.Event
tag := r.Tag.(DatabaseEvent)
if r.Err != nil {
log.Printf("Error adding event (Id: %s %s): %v", tag.Id, tag.Title, r.Err)
continue
}
event = r.Value.(*cal.Event)
updateDatabaseorSomething(tag, event.Id)
}
return
}
func updateDatabaseorSomething(ev DatabaseEvent, newCalEventId string) {
return
}
func getEventData() []DatabaseEvent {
return nil
}
func getOauthClient() *http.Client {
return nil
}
Output:
Example (Userdata) ¶
package main
import (
"github.com/jfcote87/api/googleapi"
"github.com/jfcote87/batch"
"io/ioutil"
"log"
gmail "google.golang.org/api/gmail/v1"
storage "google.golang.org/api/storage/v1"
"golang.org/x/oauth2"
"golang.org/x/oauth2/google"
"golang.org/x/oauth2/jwt"
)
func main() {
projectId, usernames, config := getInitialData()
// Retrieve the list of available buckets for each user for a given api project as well as
// profile info for each person
bsv := batch.Service{} // no need for client as individual requests will have their own authorization
storagesv, _ := storage.New(googleapi.BatchClient())
gsv, _ := gmail.New(googleapi.BatchClient())
config.Scopes = []string{gmail.MailGoogleComScope, "email", storage.DevstorageRead_onlyScope}
for _, u := range usernames {
// create new credentials for specific user
tConfig := *config
tConfig.Subject = u + "@example.com"
cred := &batch.Oauth2Credentials{TokenSource: tConfig.TokenSource(oauth2.NoContext)}
// create bucket list request
_, err := storagesv.Buckets.List(projectId).Do()
if err = bsv.Queue(err, []string{u, "Buckets"}, cred); err != nil {
log.Printf("Error adding bucklist request for %s: %v", u, err)
return
}
// create profile request
_, err = gsv.Users.GetProfile(u + "@example.com").Do()
if err = bsv.Queue(err, []string{u, "Profile"}, cred); err != nil {
log.Printf("Error adding profile request for %s: %v", u, err)
return
}
}
// execute batch
results, err := bsv.Do()
if err != nil {
log.Println(err)
return
}
// process responses
for _, r := range results {
tag := r.Tag.([]string)
if r.Err != nil {
log.Printf("Error retrieving user (%s) %s: %v", tag[0], tag[1], r.Err)
continue
}
if tag[1] == "Profile" {
profile := r.Value.(*gmail.Profile)
log.Printf("User %s profile id is %d", profile.EmailAddress, profile.HistoryId)
} else {
blist := r.Value.(*storage.Buckets)
log.Printf("User: %s", tag[0])
for _, b := range blist.Items {
log.Printf("%s", b.Name)
}
}
}
return
}
func getInitialData() (string, []string, *jwt.Config) {
jwtbytes, _ := ioutil.ReadFile("secret.json")
config, _ := google.JWTConfigFromJSON(jwtbytes)
return "XXXXXXXXXXXXXXXX", []string{"jcote", "bclinton", "gbush", "bobama"}, config
}
Output:
func (*Service) Do ¶
Do sends up to maxCalls(default 1000) in a single request. Remaining requests must be processed in subsequent calls to Do.
func (*Service) Queue ¶
func (s *Service) Queue(err error, tag interface{}, credential Credentialer) error
Queue add a new call to the Service.callQueue using the error returned from an api Do() method. If err is not a *googleapi.Call then the original error is returned.
tag provides data to be used to process the call's result.
credential provides the credentialer that will be used for the queued call. If nil, the batch.Service will use its default credentials.
example:
_, err = svCal.Events.Insert(*cal, eventData).Do()
if err = batchService.Queue(err, "tag data", batch.ServiceCredentials) {
// handle error
}
Source Files