protobq

Documentation

Overview

Package protobq marshals and unmarshals protocol buffer messages as BigQuery format.

Index

• func InferSchema(msg proto.Message) bigquery.Schema
• func Load(row []bigquery.Value, schema bigquery.Schema, message proto.Message) error
• func Marshal(msg proto.Message) (map[string]bigquery.Value, error)
• func Unmarshal(row map[string]bigquery.Value, message proto.Message) error
• type MarshalOptions
  • func (o MarshalOptions) Marshal(msg proto.Message) (map[string]bigquery.Value, error)
• type MessageLoader
  • func (m *MessageLoader) Load(row []bigquery.Value, schema bigquery.Schema) error
• type MessageSaver
  • func (m *MessageSaver) Save() (map[string]bigquery.Value, string, error)
• type SchemaOptions
  • func (o SchemaOptions) InferSchema(msg proto.Message) bigquery.Schema
• type UnmarshalOptions
  • func (o UnmarshalOptions) Load(row []bigquery.Value, schema bigquery.Schema, message proto.Message) error
  • func (o UnmarshalOptions) Unmarshal(row map[string]bigquery.Value, message proto.Message) error

Examples

• InferSchema
• Load
• Marshal
• Unmarshal

Functions

func InferSchema

func InferSchema(msg proto.Message) bigquery.Schema

InferSchema infers a BigQuery schema for the given proto.Message using default options.

Example

msg := &library.Book{}
schema := protobq.InferSchema(msg)
expected := bigquery.Schema{
	{Name: "name", Type: bigquery.StringFieldType},
	{Name: "author", Type: bigquery.StringFieldType},
	{Name: "title", Type: bigquery.StringFieldType},
	{Name: "read", Type: bigquery.BooleanFieldType},
}
fmt.Println(cmp.Equal(expected, schema))

Output:

true

func Load

func Load(row []bigquery.Value, schema bigquery.Schema, message proto.Message) error

Load the bigquery.Value list into the given proto.Message using the given bigquery.Schema. It will clear the message first before setting the fields. If it returns an error, the given message may be partially set.

Example

row := []bigquery.Value{
	"publishers/123/books/456",
	"P.L. Travers",
	"Mary Poppins",
	true,
}
schema := bigquery.Schema{
	{Name: "name", Type: bigquery.StringFieldType},
	{Name: "author", Type: bigquery.StringFieldType},
	{Name: "title", Type: bigquery.StringFieldType},
	{Name: "read", Type: bigquery.BooleanFieldType},
}
msg := &library.Book{}
if err := protobq.Load(row, schema, msg); err != nil {
	// TODO: Handle error.
}
expected := &library.Book{
	Name:   "publishers/123/books/456",
	Author: "P.L. Travers",
	Title:  "Mary Poppins",
	Read:   true,
}
fmt.Println(cmp.Equal(expected, msg, protocmp.Transform()))

Output:

true

func Marshal

func Marshal(msg proto.Message) (map[string]bigquery.Value, error)

Marshal writes the given proto.Message in BigQuery format using default options.

Example

msg := &library.Book{
	Name:   "publishers/123/books/456",
	Author: "P.L. Travers",
	Title:  "Mary Poppins",
	Read:   true,
}
row, err := protobq.Marshal(msg)
if err != nil {
	// TODO: Handle error.
}
expected := map[string]bigquery.Value{
	"name":   "publishers/123/books/456",
	"author": "P.L. Travers",
	"title":  "Mary Poppins",
	"read":   true,
}
fmt.Println(cmp.Equal(expected, row))

Output:

true

func Unmarshal

func Unmarshal(row map[string]bigquery.Value, message proto.Message) error

Unmarshal the bigquery.Value map into the given proto.Message. It will clear the message first before setting the fields. If it returns an error, the given message may be partially set.

Example

row := map[string]bigquery.Value{
	"name":   "publishers/123/books/456",
	"author": "P.L. Travers",
	"title":  "Mary Poppins",
	"read":   true,
}
msg := &library.Book{}
if err := protobq.Unmarshal(row, msg); err != nil {
	// TODO: Handle error.
}
expected := &library.Book{
	Name:   "publishers/123/books/456",
	Author: "P.L. Travers",
	Title:  "Mary Poppins",
	Read:   true,
}
fmt.Println(cmp.Equal(expected, msg, protocmp.Transform()))

Output:

true

Types

type MarshalOptions

type MarshalOptions struct {
	// Schema contains the schema options.
	Schema SchemaOptions
}

MarshalOptions is a configurable BigQuery format marshaler.

func (MarshalOptions) Marshal

func (o MarshalOptions) Marshal(msg proto.Message) (map[string]bigquery.Value, error)

Marshal marshals the given proto.Message in the BigQuery format using options in MarshalOptions.

type MessageLoader

type MessageLoader struct {
	// Options to use for unmarshaling the Message.
	Options UnmarshalOptions

	// Message to load.
	Message proto.Message
}

MessageLoader implements bigquery.ValueLoader for a proto.Message. The message is converted from a BigQuery row using the provided UnmarshalOptions.

func (*MessageLoader) Load

func (m *MessageLoader) Load(row []bigquery.Value, schema bigquery.Schema) error

Load implements bigquery.ValueLoader.

type MessageSaver

type MessageSaver struct {
	// Options to use for marshaling the Message.
	Options MarshalOptions

	// InsertID governs the best-effort deduplication feature of
	// BigQuery streaming inserts.
	//
	// If the InsertID is empty, a random InsertID will be generated by
	// this library to facilitate deduplication.
	//
	// If the InsertID is set to the sentinel value bigquery.NoDedupeID, an InsertID
	// is not sent.
	//
	// For all other non-empty values, BigQuery will use the provided
	// value for best-effort deduplication.
	InsertID string

	// Message to save.
	Message proto.Message
}

MessageSaver implements bigquery.ValueSaver for a proto.Message. The message is converted to a BigQuery row using the provided MarshalOptions.

func (*MessageSaver) Save

func (m *MessageSaver) Save() (map[string]bigquery.Value, string, error)

Save implements bigquery.ValueSaver.

type SchemaOptions

type SchemaOptions struct{}

SchemaOptions contains configuration options for BigQuery schema inference.

func (SchemaOptions) InferSchema

func (o SchemaOptions) InferSchema(msg proto.Message) bigquery.Schema

InferSchema infers a BigQuery schema for the given proto.Message using options in MarshalOptions.

type UnmarshalOptions

type UnmarshalOptions struct {
	// Schema contains the schema options.
	Schema SchemaOptions

	// If AllowPartial is set, input for messages that will result in missing
	// required fields will not return an error.
	AllowPartial bool

	// If DiscardUnknown is set, unknown fields are ignored.
	DiscardUnknown bool
}

UnmarshalOptions is a configurable BigQuery format parser.

func (UnmarshalOptions) Load

func (o UnmarshalOptions) Load(row []bigquery.Value, schema bigquery.Schema, message proto.Message) error

Load the bigquery.Value list into the given proto.Message using the given bigquery.Schema using options in UnmarshalOptions object. It will clear the message first before setting the fields. If it returns an error, the given message may be partially set.

func (UnmarshalOptions) Unmarshal

func (o UnmarshalOptions) Unmarshal(row map[string]bigquery.Value, message proto.Message) error

Unmarshal reads the given BigQuery row and populates the given proto.Message using options in UnmarshalOptions object. It will clear the message first before setting the fields. If it returns an error, the given message may be partially set.