util

Documentation

Index

• func TotalArraySize(arr arrow.Array) int64
• func TotalRecordSize(record arrow.Record) int64
• func WithEnumHandler(enumHandler ProtobufTypeHandler) option
• func WithExclusionPolicy(ex func(pfr *ProtobufFieldReflection) bool) option
• func WithFieldNameFormatter(formatter func(str string) string) option
• func WithOneOfHandler(oneOfHandler ProtobufTypeHandler) option
• type ProtobufFieldReflection
  • func (pfr *ProtobufFieldReflection) GetDescriptor() protoreflect.FieldDescriptor
• type ProtobufMessageFieldReflection
  • func (f ProtobufMessageFieldReflection) AppendValueOrNull(b array.Builder, mem memory.Allocator) error
• type ProtobufMessageReflection
  • func NewProtobufMessageReflection(msg proto.Message, options ...option) *ProtobufMessageReflection
  • func (msg ProtobufMessageReflection) Record(mem memory.Allocator) arrow.Record
  • func (msg ProtobufMessageReflection) Schema() *arrow.Schema
• type ProtobufTypeHandler

Functions

func TotalArraySize

func TotalArraySize(arr arrow.Array) int64

TotalArraySize returns the sum of the number of bytes in each buffer referenced by the Array.

func TotalRecordSize

func TotalRecordSize(record arrow.Record) int64

TotalRecordSize return the sum of bytes in each buffer referenced by the Record.

func WithEnumHandler

func WithEnumHandler(enumHandler ProtobufTypeHandler) option

WithEnumHandler is an option for a ProtobufMessageReflection WithEnumHandler enables customisation of the protobuf Enum type in the arrow schema By default, the Enums are mapped to arrow.Dictionary

func WithExclusionPolicy

func WithExclusionPolicy(ex func(pfr *ProtobufFieldReflection) bool) option

WithExclusionPolicy is an option for a ProtobufMessageReflection WithExclusionPolicy acts as a deny filter on the fields of a protobuf message i.e. prevents them from being included in the schema. A use case for this is to exclude fields containing PII.

func WithFieldNameFormatter

func WithFieldNameFormatter(formatter func(str string) string) option

WithFieldNameFormatter is an option for a ProtobufMessageReflection WithFieldNameFormatter enables customisation of the field names in the arrow schema By default, the field names are taken from the protobuf message (.proto file)

func WithOneOfHandler

func WithOneOfHandler(oneOfHandler ProtobufTypeHandler) option

WithOneOfHandler is an option for a ProtobufMessageReflection WithOneOfHandler enables customisation of the protobuf oneOf type in the arrow schema By default, the oneOfs are mapped to separate columns

Types

type ProtobufFieldReflection

type ProtobufFieldReflection struct {
	arrow.Field
	// contains filtered or unexported fields
}

ProtobufFieldReflection represents the metadata and values of a protobuf field

func (*ProtobufFieldReflection) GetDescriptor

func (pfr *ProtobufFieldReflection) GetDescriptor() protoreflect.FieldDescriptor

type ProtobufMessageFieldReflection

type ProtobufMessageFieldReflection struct {
	arrow.Field
	// contains filtered or unexported fields
}

ProtobufMessageFieldReflection links together the message and it's fields

func (ProtobufMessageFieldReflection) AppendValueOrNull

func (f ProtobufMessageFieldReflection) AppendValueOrNull(b array.Builder, mem memory.Allocator) error

AppendValueOrNull add the value of a protobuf field to an arrow array builder

type ProtobufMessageReflection

type ProtobufMessageReflection struct {
	// contains filtered or unexported fields
}

ProtobufMessageReflection represents the metadata and values of a protobuf message

func NewProtobufMessageReflection

func NewProtobufMessageReflection(msg proto.Message, options ...option) *ProtobufMessageReflection

NewProtobufMessageReflection initialises a ProtobufMessageReflection can be used to convert a protobuf message into an arrow Record

func (ProtobufMessageReflection) Record

func (msg ProtobufMessageReflection) Record(mem memory.Allocator) arrow.Record

Record returns an arrow.Record for a protobuf message

func (ProtobufMessageReflection) Schema

func (msg ProtobufMessageReflection) Schema() *arrow.Schema

Schema returns an arrow.Schema representing a protobuf message

type ProtobufTypeHandler

type ProtobufTypeHandler int

ProtobufTypeHandler provides options on how protobuf fields should be handled in the conversion to arrow

const (
	// OneOfNull means do not wrap oneOfs in a union, they are treated as separate fields
	OneOfNull ProtobufTypeHandler = iota
	// OneOfDenseUnion maps the protobuf OneOf to an arrow.DENSE_UNION
	OneOfDenseUnion
	// EnumNumber uses the Enum numeric value
	EnumNumber
	// EnumValue uses the Enum string value
	EnumValue
	// EnumDictionary uses both the numeric and string value and maps to an arrow.Dictionary
	EnumDictionary
)