arrowpb

README

arrowpb

Convert Apache Arrow records to Protocol Buffers

Examples

For examples, see the arrowpb-example repository.

Author

arrowpb is developed by @TFMV.

License

This project is licensed under the MIT License. See the LICENSE file for details.

Documentation

Index

• func ArrowReaderToProtos(ctx context.Context, reader array.RecordReader, ...) ([][]byte, error)
• func ArrowSchemaToFileDescriptorProto(schema *arrow.Schema, packageName, messagePrefix string, cfg *ConvertConfig) (*descriptorpb.FileDescriptorProto, error)
• func CompileFileDescriptorProto(fdp *descriptorpb.FileDescriptorProto) (protoreflect.FileDescriptor, error)
• func CompileFileDescriptorProtoWithRetry(fdp *descriptorpb.FileDescriptorProto) (protoreflect.FileDescriptor, error)
• func ConvertInParallel(ctx context.Context, record arrow.Record, ...) ([][]byte, error)
• func CreateArrowRecord() (array.RecordReader, error)
• func ExtractArrowValue(col arrow.Array, rowIndex int) interface{}
• func FormatArrowJSON(reader array.RecordReader, output io.Writer) error
• func GenerateUniqueMessageName(prefix string) string
• func GetTopLevelMessageDescriptor(fd protoreflect.FileDescriptor) (protoreflect.MessageDescriptor, error)
• func RecordToDynamicProtos(rec arrow.Record, msgDesc protoreflect.MessageDescriptor, cfg *ConvertConfig) ([][]byte, error)
• func RowToDynamicProto(record arrow.Record, msgDesc protoreflect.MessageDescriptor, rowIndex int, ...) (*dynamicpb.Message, error)
• type ConvertConfig

Functions

func ArrowReaderToProtos

func ArrowReaderToProtos(ctx context.Context, reader array.RecordReader, msgDesc protoreflect.MessageDescriptor, cfg *ConvertConfig) ([][]byte, error)

func ArrowSchemaToFileDescriptorProto

func ArrowSchemaToFileDescriptorProto(schema *arrow.Schema, packageName, messagePrefix string, cfg *ConvertConfig) (*descriptorpb.FileDescriptorProto, error)

func CompileFileDescriptorProto

func CompileFileDescriptorProto(fdp *descriptorpb.FileDescriptorProto) (protoreflect.FileDescriptor, error)

CompileFileDescriptorProto merges your generated FileDescriptorProto with the WKT descriptors so that references to google.protobuf.Timestamp, etc., can be resolved.

func CompileFileDescriptorProtoWithRetry

func CompileFileDescriptorProtoWithRetry(fdp *descriptorpb.FileDescriptorProto) (protoreflect.FileDescriptor, error)

CompileFileDescriptorProtoWithRetry wraps the above in an exponential backoff.

func ConvertInParallel

func ConvertInParallel(ctx context.Context, record arrow.Record, msgDesc protoreflect.MessageDescriptor, concurrency int, cfg *ConvertConfig) ([][]byte, error)

ConvertInParallel processes a single Arrow Record in parallel, chunking row ranges.

func CreateArrowRecord

func CreateArrowRecord() (array.RecordReader, error)

func ExtractArrowValue

func ExtractArrowValue(col arrow.Array, rowIndex int) interface{}

func FormatArrowJSON

func FormatArrowJSON(reader array.RecordReader, output io.Writer) error

func GenerateUniqueMessageName

func GenerateUniqueMessageName(prefix string) string

func GetTopLevelMessageDescriptor

func GetTopLevelMessageDescriptor(fd protoreflect.FileDescriptor) (protoreflect.MessageDescriptor, error)

GetTopLevelMessageDescriptor fetches the first message from a compiled FileDescriptor.

func RecordToDynamicProtos

func RecordToDynamicProtos(rec arrow.Record, msgDesc protoreflect.MessageDescriptor, cfg *ConvertConfig) ([][]byte, error)

func RowToDynamicProto

func RowToDynamicProto(record arrow.Record, msgDesc protoreflect.MessageDescriptor, rowIndex int, cfg *ConvertConfig) (*dynamicpb.Message, error)

Types

type ConvertConfig

type ConvertConfig struct {
	UseWellKnownTimestamps bool // arrow.TIMESTAMP => google.protobuf.Timestamp
	UseProto2Syntax        bool // changes FileDescriptorProto syntax to "proto2"
	UseWrapperTypes        bool // arrow scalars => google.protobuf.*Value
	MapDictionariesToEnums bool // dictionary => enum

	// DescriptorCache can store repeated schemas => reuse of the same descriptor
	DescriptorCache sync.Map
}

ConvertConfig allows fine-grained control over Arrow => Protobuf schema generation.