protobq

package
v0.24.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jan 18, 2023 License: MIT Imports: 18 Imported by: 4

Documentation

Overview

Package protobq implements saving and loading protobuf messages to and from BigQuery.

Index

Examples

Constants

This section is empty.

Variables

This section is empty.

Functions

func InferSchema 

func InferSchema(msg proto.Message) bigquery.Schema

InferSchema infers a BigQuery schema for the given proto.Message using default options.

Example 
msg := &library.Book{}
schema := protobq.InferSchema(msg)
expected := bigquery.Schema{
	{Name: "name", Type: bigquery.StringFieldType},
	{Name: "author", Type: bigquery.StringFieldType},
	{Name: "title", Type: bigquery.StringFieldType},
	{Name: "read", Type: bigquery.BooleanFieldType},
}
fmt.Println(cmp.Equal(expected, schema))

Output:

true

func Load 

func Load(bqMessage []bigquery.Value, bqSchema bigquery.Schema, message proto.Message) error

Load the bigquery.Value list into the given proto.Message using the given bigquery.Schema. It will clear the message first before setting the fields. If it returns an error, the given message may be partially set.

Example 
row := []bigquery.Value{
	"publishers/123/books/456",
	"P.L. Travers",
	"Mary Poppins",
	true,
}
schema := bigquery.Schema{
	{Name: "name", Type: bigquery.StringFieldType},
	{Name: "author", Type: bigquery.StringFieldType},
	{Name: "title", Type: bigquery.StringFieldType},
	{Name: "read", Type: bigquery.BooleanFieldType},
}
msg := &library.Book{}
if err := protobq.Load(row, schema, msg); err != nil {
	// TODO: Handle error.
}
expected := &library.Book{
	Name:   "publishers/123/books/456",
	Author: "P.L. Travers",
	Title:  "Mary Poppins",
	Read:   true,
}
fmt.Println(cmp.Equal(expected, msg, protocmp.Transform()))

Output:

true

func Marshal 

func Marshal(msg proto.Message) (map[string]bigquery.Value, error)

Marshal writes the given proto.Message in BigQuery format using default options.

Example 
msg := &library.Book{
	Name:   "publishers/123/books/456",
	Author: "P.L. Travers",
	Title:  "Mary Poppins",
	Read:   true,
}
row, err := protobq.Marshal(msg)
if err != nil {
	// TODO: Handle error.
}
expected := map[string]bigquery.Value{
	"name":   "publishers/123/books/456",
	"author": "P.L. Travers",
	"title":  "Mary Poppins",
	"read":   true,
}
fmt.Println(cmp.Equal(expected, row))

Output:

true

func Unmarshal 

func Unmarshal(bqMessage map[string]bigquery.Value, message proto.Message) error

Unmarshal the bigquery.Value map into the given proto.Message. It will clear the message first before setting the fields. If it returns an error, the given message may be partially set.

Example 
row := map[string]bigquery.Value{
	"name":   "publishers/123/books/456",
	"author": "P.L. Travers",
	"title":  "Mary Poppins",
	"read":   true,
}
msg := &library.Book{}
if err := protobq.Unmarshal(row, msg); err != nil {
	// TODO: Handle error.
}
expected := &library.Book{
	Name:   "publishers/123/books/456",
	Author: "P.L. Travers",
	Title:  "Mary Poppins",
	Read:   true,
}
fmt.Println(cmp.Equal(expected, msg, protocmp.Transform()))

Output:

true

Types

type MarshalOptions 

type MarshalOptions struct {
	// Schema contains the schema options.
	Schema SchemaOptions
}

MarshalOptions is a configurable BigQuery format marshaler.

func (MarshalOptions) Marshal 

func (o MarshalOptions) Marshal(msg proto.Message) (map[string]bigquery.Value, error)

Marshal marshals the given proto.Message in the BigQuery format using options in MarshalOptions.

type MessageLoader 

type MessageLoader struct {
	// Options to use for unmarshaling the Message.
	Options UnmarshalOptions

	// Message to load.
	Message proto.Message
}

MessageLoader implements bigquery.ValueLoader for a proto.Message. The message is converted from a BigQuery row using the provided UnmarshalOptions.

Example 
package main

import (
	"context"
	"errors"
	"fmt"

	"cloud.google.com/go/bigquery"
	"go.einride.tech/protobuf-bigquery/encoding/protobq"

	publicv1 "go.einride.tech/protobuf-bigquery/internal/examples/proto/gen/go/einride/bigquery/public/v1"
	"google.golang.org/api/iterator"
	"google.golang.org/protobuf/encoding/prototext"
)

func main() {
	ctx := context.Background()
	// Read from the public "film locations" BigQuery dataset into a proto message.
	const (
		project = "bigquery-public-data"
		dataset = "san_francisco_film_locations"
		table   = "film_locations"
	)
	// Connect to BigQuery.
	client, err := bigquery.NewClient(ctx, project)
	if err != nil {
		panic(err) // TODO: Handle error.
	}
	// Load BigQuery rows into a FilmLocation message.
	messageLoader := &protobq.MessageLoader{
		Message: &publicv1.FilmLocation{},
	}
	// Iterate rows in table.
	rowIterator := client.Dataset(dataset).Table(table).Read(ctx)
	for {
		// Load next row into the FilmLocation message.
		if err := rowIterator.Next(messageLoader); err != nil {
			if errors.Is(err, iterator.Done) {
				break
			}
			panic(err) // TODO: Handle error.
		}
		// Print the message.
		fmt.Println(prototext.Format(messageLoader.Message))
	}
}

Output:

func (*MessageLoader) Load 

func (m *MessageLoader) Load(row []bigquery.Value, schema bigquery.Schema) error

Load implements bigquery.ValueLoader.

type MessageSaver 

type MessageSaver struct {
	// Options to use for marshaling the Message.
	Options MarshalOptions

	// InsertID governs the best-effort deduplication feature of
	// BigQuery streaming inserts.
	//
	// If the InsertID is empty, a random InsertID will be generated by
	// this library to facilitate deduplication.
	//
	// If the InsertID is set to the sentinel value bigquery.NoDedupeID, an InsertID
	// is not sent.
	//
	// For all other non-empty values, BigQuery will use the provided
	// value for best-effort deduplication.
	InsertID string

	// Message to save.
	Message proto.Message
}

MessageSaver implements bigquery.ValueSaver for a proto.Message. The message is converted to a BigQuery row using the provided MarshalOptions.

Example 
package main

import (
	"context"
	"flag"
	"strconv"

	"cloud.google.com/go/bigquery"
	"go.einride.tech/protobuf-bigquery/encoding/protobq"

	publicv1 "go.einride.tech/protobuf-bigquery/internal/examples/proto/gen/go/einride/bigquery/public/v1"
)

func main() {
	ctx := context.Background()
	// Write protobuf messages to a BigQuery table.
	projectID := flag.String("project", "", "BigQuery project to write to.")
	datasetID := flag.String("dataset", "", "BigQuery dataset to write to.")
	tableID := flag.String("table", "", "BigQuery table to write to.")
	create := flag.Bool("create", false, "Flag indicating whether to create the table.")
	flag.Parse()
	// Connect to BigQuery.
	client, err := bigquery.NewClient(ctx, *projectID)
	if err != nil {
		panic(err) // TODO: Handle error.
	}
	table := client.Dataset(*datasetID).Table(*tableID)
	// Create the table by inferring the BigQuery schema from the protobuf schema.
	if *create {
		if err := table.Create(ctx, &bigquery.TableMetadata{
			Schema: protobq.InferSchema(&publicv1.FilmLocation{}),
		}); err != nil {
			panic(err) // TODO: Handle error.
		}
	}
	// Insert the protobuf messages.
	inserter := table.Inserter()
	for i, filmLocation := range []*publicv1.FilmLocation{
		{Title: "Dark Passage", ReleaseYear: 1947, Locations: "Filbert Steps"},
		{Title: "D.O.A", ReleaseYear: 1950, Locations: "Union Square"},
		{Title: "Flower Drum Song", ReleaseYear: 1961, Locations: "Chinatown"},
	} {
		if err := inserter.Put(ctx, &protobq.MessageSaver{
			Message:  filmLocation,
			InsertID: strconv.Itoa(i), // include an optional insert ID
		}); err != nil {
			panic(err) // TODO: Handle error.
		}
	}
}

Output:

func (*MessageSaver) Save 

func (m *MessageSaver) Save() (map[string]bigquery.Value, string, error)

Save implements bigquery.ValueSaver.

type SchemaOptions 

type SchemaOptions struct {
	// UseEnumNumbers converts enum values to INTEGER types.
	UseEnumNumbers bool
	// UseDateTimeWithoutOffset converts google.type.DateTime values to DATETIME, discarding the optional time offset.
	UseDateTimeWithoutOffset bool
	// UseOneofFields adds an extra STRING field for oneof fields with the name of the oneof,
	// containing the name of the field that is set.
	UseOneofFields bool
	// UseModeFromFieldBehavior sets the mode of a field to REQUIRED if the field is defined with REQUIRED behavior
	// in proto.
	UseModeFromFieldBehavior bool
	// UseProtoCommentsAsDescription use the proto comments to populate the description of the BigQuery field
	UseProtoCommentsAsDescription bool
	// UseJSONStructs use JSON type instead of string for STRUCT type
	UseJSONStructs bool
}

SchemaOptions contains configuration options for BigQuery schema inference.

func (SchemaOptions) InferMessageSchema added in v0.18.0 

func (o SchemaOptions) InferMessageSchema(msg protoreflect.MessageDescriptor) bigquery.Schema

InferMessageSchema infers the BigQuery schema for the given protoreflect.MessageDescriptor.

func (SchemaOptions) InferSchema 

func (o SchemaOptions) InferSchema(msg proto.Message) bigquery.Schema

InferSchema infers a BigQuery schema for the given proto.Message using options in MarshalOptions.

type UnmarshalOptions 

type UnmarshalOptions struct {
	// Schema contains the schema options.
	Schema SchemaOptions

	// If AllowPartial is set, input for messages that will result in missing
	// required fields will not return an error.
	AllowPartial bool

	// If DiscardUnknown is set, unknown fields are ignored.
	DiscardUnknown bool
}

UnmarshalOptions is a configurable BigQuery format parser.

func (UnmarshalOptions) Load 

func (o UnmarshalOptions) Load(bqMessage []bigquery.Value, bqSchema bigquery.Schema, message proto.Message) error

Load the bigquery.Value list into the given proto.Message using the given bigquery.Schema using options in UnmarshalOptions object. It will clear the message first before setting the fields. If it returns an error, the given message may be partially set.

func (UnmarshalOptions) Unmarshal 

func (o UnmarshalOptions) Unmarshal(bqMessage map[string]bigquery.Value, message proto.Message) error

Unmarshal reads the given BigQuery row and populates the given proto.Message using options in UnmarshalOptions object. It will clear the message first before setting the fields. If it returns an error, the given message may be partially set.

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.