arrow

package
v0.0.0-...-cafc5b9 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Aug 16, 2023 License: Apache-2.0 Imports: 28 Imported by: 0

Documentation

Overview

Package arrow contains common types and functions used to convert OTLP entities into their Arrow representation.

Index

Constants

View Source 
const (
	ColorReset = "\033[0m"
	Green      = "\033[32m"
	Cyan       = "\033[36m"
	Grey       = "\033[90m"
)
View Source 
const (
	StrCode    int8 = 0
	I64Code    int8 = 1
	F64Code    int8 = 2
	BoolCode   int8 = 3
	BinaryCode int8 = 4
	CborCode   int8 = 5
)

Constants used to identify the type of value in the union.

View Source 
const (
	// ParentIdNoEncoding stores the parent ID as is.
	ParentIdNoEncoding = iota
	// ParentIdDeltaEncoding stores the parent ID as a delta from the previous
	// parent ID.
	ParentIdDeltaEncoding
	// ParentIdDeltaGroupEncoding stores the parent ID as a delta from the
	// previous parent ID in the same group. A group is defined by the
	// combination Key and Value.
	ParentIdDeltaGroupEncoding
)

ParentID encodings

View Source 
const (
	MetadataType = "type"

	BoolType       = "bool"
	IntType        = "i64"
	IntDictType    = "i64_dict"
	DoubleType     = "f64"
	StringType     = "str"
	StringDictType = "str_dict"
	BytesType      = "bytes"
	BytesDictType  = "bytes_dict"
	CborType       = "cbor"
	CborDictType   = "cbor_dict"
)

Variables

View Source 
var (
	// KDT is the Arrow key data type.
	KDT = arrow.BinaryTypes.String

	// AttributesDT is the Arrow attribute data type.
	AttributesDT = arrow.MapOfWithMetadata(
		KDT, schema.Metadata(schema.Dictionary8),
		AnyValueDT, schema.Metadata(),
	)
)

Arrow data types used to build the attribute map.

View Source 
var (
	// AttrsSchema32 is the Arrow schema used to represent attribute records
	// with 16-bit Parent IDs.
	// This schema doesn't use the Arrow union type to make Parquet conversion
	// more direct.
	AttrsSchema32 = arrow.NewSchema([]arrow.Field{
		{Name: constants.ParentID, Type: arrow.PrimitiveTypes.Uint32, Metadata: schema.Metadata(schema.Dictionary8)},
		{Name: constants.AttributeKey, Type: arrow.BinaryTypes.String, Metadata: schema.Metadata(schema.Dictionary8)},
		{Name: constants.AttributeType, Type: arrow.PrimitiveTypes.Uint8},
		{Name: constants.AttributeStr, Type: arrow.BinaryTypes.String, Metadata: schema.Metadata(schema.Dictionary16)},
		{Name: constants.AttributeInt, Type: arrow.PrimitiveTypes.Int64, Metadata: schema.Metadata(schema.Dictionary16), Nullable: true},
		{Name: constants.AttributeDouble, Type: arrow.PrimitiveTypes.Float64, Nullable: true},
		{Name: constants.AttributeBool, Type: arrow.FixedWidthTypes.Boolean, Nullable: true},
		{Name: constants.AttributeBytes, Type: arrow.BinaryTypes.Binary, Metadata: schema.Metadata(schema.Dictionary16), Nullable: true},
		{Name: constants.AttributeSer, Type: arrow.BinaryTypes.Binary, Metadata: schema.Metadata(schema.Dictionary16), Nullable: true},
	}, nil)

	// DeltaEncodedAttrsSchema32 is the Arrow schema used to represent attribute records
	// with 16-bit Parent IDs that are delta encoded.
	// This schema doesn't use the Arrow union type to make Parquet conversion
	// more direct.
	DeltaEncodedAttrsSchema32 = arrow.NewSchema([]arrow.Field{
		{Name: constants.ParentID, Type: arrow.PrimitiveTypes.Uint32, Metadata: schema.Metadata(schema.Dictionary8, schema.DeltaEncoding)},
		{Name: constants.AttributeKey, Type: arrow.BinaryTypes.String, Metadata: schema.Metadata(schema.Dictionary8)},
		{Name: constants.AttributeType, Type: arrow.PrimitiveTypes.Uint8},
		{Name: constants.AttributeStr, Type: arrow.BinaryTypes.String, Metadata: schema.Metadata(schema.Dictionary16)},
		{Name: constants.AttributeInt, Type: arrow.PrimitiveTypes.Int64, Metadata: schema.Metadata(schema.Dictionary16), Nullable: true},
		{Name: constants.AttributeDouble, Type: arrow.PrimitiveTypes.Float64, Nullable: true},
		{Name: constants.AttributeBool, Type: arrow.FixedWidthTypes.Boolean, Nullable: true},
		{Name: constants.AttributeBytes, Type: arrow.BinaryTypes.Binary, Metadata: schema.Metadata(schema.Dictionary16), Nullable: true},
		{Name: constants.AttributeSer, Type: arrow.BinaryTypes.Binary, Metadata: schema.Metadata(schema.Dictionary16), Nullable: true},
	}, nil)
)
View Source 
var (
	ErrBuilderAlreadyReleased = errors.New("builder already released")
	ErrInvalidResourceID      = errors.New("invalid resource ID")
	ErrInvalidScopeID         = errors.New("invalid scope ID")
)
View Source 
var (
	// AnyValueDT is an Arrow Data Type representing an OTLP Any Value.
	// Any values are represented as a sparse union of the following variants:
	// str, i64, f64, bool, binary.
	//
	// Note: str, i64, binary, and cbor are dictionary encoded by default and
	// will fall back to a non-dictionary encoding when the dictionary index
	// overflowed.
	AnyValueDT = arrow.SparseUnionOf([]arrow.Field{
		{Name: "str", Type: arrow.BinaryTypes.String, Metadata: schema.Metadata(schema.Dictionary16)},
		{Name: "i64", Type: arrow.PrimitiveTypes.Int64, Metadata: schema.Metadata(schema.Optional, schema.Dictionary16)},
		{Name: "f64", Type: arrow.PrimitiveTypes.Float64, Metadata: schema.Metadata(schema.Optional)},
		{Name: "bool", Type: arrow.FixedWidthTypes.Boolean, Metadata: schema.Metadata(schema.Optional)},
		{Name: "binary", Type: arrow.BinaryTypes.Binary, Metadata: schema.Metadata(schema.Optional, schema.Dictionary16)},
		{Name: "cbor", Type: arrow.BinaryTypes.Binary, Metadata: schema.Metadata(schema.Optional, schema.Dictionary16)},
	}, []int8{
		StrCode,
		I64Code,
		F64Code,
		BoolCode,
		BinaryCode,
		CborCode,
	})
)
View Source 
var (
	// AttrsSchema16 is the Arrow schema used to represent attribute records
	// with 16-bit Parent IDs.
	// This schema doesn't use the Arrow union type to make Parquet conversion
	// more direct.
	AttrsSchema16 = arrow.NewSchema([]arrow.Field{
		{Name: constants.ParentID, Type: arrow.PrimitiveTypes.Uint16},
		{Name: constants.AttributeKey, Type: arrow.BinaryTypes.String, Metadata: schema.Metadata(schema.Dictionary8)},
		{Name: constants.AttributeType, Type: arrow.PrimitiveTypes.Uint8},
		{Name: constants.AttributeStr, Type: arrow.BinaryTypes.String, Metadata: schema.Metadata(schema.Dictionary16)},
		{Name: constants.AttributeInt, Type: arrow.PrimitiveTypes.Int64, Metadata: schema.Metadata(schema.Dictionary16), Nullable: true},
		{Name: constants.AttributeDouble, Type: arrow.PrimitiveTypes.Float64, Nullable: true},
		{Name: constants.AttributeBool, Type: arrow.FixedWidthTypes.Boolean, Nullable: true},
		{Name: constants.AttributeBytes, Type: arrow.BinaryTypes.Binary, Metadata: schema.Metadata(schema.Dictionary16), Nullable: true},
		{Name: constants.AttributeSer, Type: arrow.BinaryTypes.Binary, Metadata: schema.Metadata(schema.Dictionary16), Nullable: true},
	}, nil)
)
View Source 
var (
	PayloadTypes = payloadTypes{
		Metrics: &PayloadType{
			prefix:      "metrics",
			payloadType: colarspb.ArrowPayloadType_METRICS,
		},
		Logs: &PayloadType{
			prefix:      "logs",
			payloadType: colarspb.ArrowPayloadType_LOGS,
		},
		Spans: &PayloadType{
			prefix:      "spans",
			payloadType: colarspb.ArrowPayloadType_SPANS,
		},
		ResourceAttrs: &PayloadType{
			prefix:      "resource-attrs",
			payloadType: colarspb.ArrowPayloadType_RESOURCE_ATTRS,
		},
		ScopeAttrs: &PayloadType{
			prefix:      "scope-attrs",
			payloadType: colarspb.ArrowPayloadType_SCOPE_ATTRS,
		},
		NumberDataPoints: &PayloadType{
			prefix:      "number-dps",
			payloadType: colarspb.ArrowPayloadType_NUMBER_DATA_POINTS,
		},
		NumberDataPointAttrs: &PayloadType{
			prefix:      "number-dp-attrs",
			payloadType: colarspb.ArrowPayloadType_NUMBER_DP_ATTRS,
		},
		NumberDataPointExemplars: &PayloadType{
			prefix:      "number-dp-exemplars",
			payloadType: colarspb.ArrowPayloadType_NUMBER_DP_EXEMPLARS,
		},
		NumberDataPointExemplarAttrs: &PayloadType{
			prefix:      "number-dp-exemplar-attrs",
			payloadType: colarspb.ArrowPayloadType_NUMBER_DP_EXEMPLAR_ATTRS,
		},
		Summary: &PayloadType{
			prefix:      "summary-dps",
			payloadType: colarspb.ArrowPayloadType_SUMMARY_DATA_POINTS,
		},
		SummaryAttrs: &PayloadType{
			prefix:      "summary-dp-attrs",
			payloadType: colarspb.ArrowPayloadType_SUMMARY_DP_ATTRS,
		},
		Histogram: &PayloadType{
			prefix:      "histogram-dps",
			payloadType: colarspb.ArrowPayloadType_HISTOGRAM_DATA_POINTS,
		},
		HistogramAttrs: &PayloadType{
			prefix:      "histogram-dp-attrs",
			payloadType: colarspb.ArrowPayloadType_HISTOGRAM_DP_ATTRS,
		},
		HistogramExemplars: &PayloadType{
			prefix:      "histogram-dp-exemplars",
			payloadType: colarspb.ArrowPayloadType_HISTOGRAM_DP_EXEMPLARS,
		},
		HistogramExemplarAttrs: &PayloadType{
			prefix:      "histogram-dp-exemplar-attrs",
			payloadType: colarspb.ArrowPayloadType_HISTOGRAM_DP_EXEMPLAR_ATTRS,
		},
		ExpHistogram: &PayloadType{
			prefix:      "exp-histogram-dps",
			payloadType: colarspb.ArrowPayloadType_EXP_HISTOGRAM_DATA_POINTS,
		},
		ExpHistogramAttrs: &PayloadType{
			prefix:      "exp-histogram-dp-attrs",
			payloadType: colarspb.ArrowPayloadType_EXP_HISTOGRAM_DP_ATTRS,
		},
		ExpHistogramExemplars: &PayloadType{
			prefix:      "exp-histogram-dp-exemplars",
			payloadType: colarspb.ArrowPayloadType_EXP_HISTOGRAM_DP_EXEMPLARS,
		},
		ExpHistogramExemplarAttrs: &PayloadType{
			prefix:      "exp-histogram-dp-exemplar-attrs",
			payloadType: colarspb.ArrowPayloadType_EXP_HISTOGRAM_DP_EXEMPLAR_ATTRS,
		},
		LogRecordAttrs: &PayloadType{
			prefix:      "logs-attrs",
			payloadType: colarspb.ArrowPayloadType_LOG_ATTRS,
		},
		SpanAttrs: &PayloadType{
			prefix:      "span-attrs",
			payloadType: colarspb.ArrowPayloadType_SPAN_ATTRS,
		},
		Event: &PayloadType{
			prefix:      "span-event",
			payloadType: colarspb.ArrowPayloadType_SPAN_EVENTS,
		},
		EventAttrs: &PayloadType{
			prefix:      "span-event-attrs",
			payloadType: colarspb.ArrowPayloadType_SPAN_EVENT_ATTRS,
		},
		Link: &PayloadType{
			prefix:      "span-link",
			payloadType: colarspb.ArrowPayloadType_SPAN_LINKS,
		},
		LinkAttrs: &PayloadType{
			prefix:      "span-link-attrs",
			payloadType: colarspb.ArrowPayloadType_SPAN_LINK_ATTRS,
		},
	}
)
View Source 
var (
	ResourceDT = arrow.StructOf([]arrow.Field{
		{
			Name:     constants.ID,
			Type:     arrow.PrimitiveTypes.Uint16,
			Metadata: schema.Metadata(schema.DeltaEncoding),
			Nullable: true,
		},
		{
			Name:     constants.SchemaUrl,
			Type:     arrow.BinaryTypes.String,
			Metadata: schema.Metadata(schema.Dictionary8),
			Nullable: true,
		},
		{
			Name:     constants.DroppedAttributesCount,
			Type:     arrow.PrimitiveTypes.Uint32,
			Nullable: true,
		},
	}...)
)

ResourceDT is the Arrow Data Type describing a resource.

View Source 
var (
	ScopeDT = arrow.StructOf([]arrow.Field{
		{Name: constants.ID, Type: arrow.PrimitiveTypes.Uint16, Metadata: acommon.Metadata(acommon.DeltaEncoding), Nullable: true},
		{Name: constants.Name, Type: arrow.BinaryTypes.String, Metadata: acommon.Metadata(acommon.Dictionary8), Nullable: true},
		{Name: constants.Version, Type: arrow.BinaryTypes.String, Metadata: acommon.Metadata(acommon.Dictionary8), Nullable: true},
		{Name: constants.DroppedAttributesCount, Type: arrow.PrimitiveTypes.Uint32, Nullable: true},
	}...)
)

ScopeDT is the Arrow Data Type describing a scope.

Functions

func Compare 

func Compare(a, b *pcommon.Value) int

func Equal 

func Equal(a, b *pcommon.Value) bool

func IsLess 

func IsLess(a, b *pcommon.Value) bool

func RequireNoError 

func RequireNoError(err error)

func WithSort 

func WithSort() func(*Options)

func WithStats 

func WithStats() func(*Options)

Types

type AnyValueBuilder 

type AnyValueBuilder struct {
	// contains filtered or unexported fields
}

AnyValueBuilder is a helper to build an Arrow array containing a collection of OTLP Any Value.

func AnyValueBuilderFrom 

func AnyValueBuilderFrom(av *builder.SparseUnionBuilder) *AnyValueBuilder

AnyValueBuilderFrom creates a new AnyValueBuilder from an existing SparseUnionBuilder.

func (*AnyValueBuilder) Append 

func (b *AnyValueBuilder) Append(av *pcommon.Value) error

Append appends a new any value to the builder.

func (*AnyValueBuilder) Build 

func (b *AnyValueBuilder) Build() (*array.SparseUnion, error)

Build builds the "any value" Arrow array.

Once the returned array is no longer needed, Release() must be called to free the memory allocated by the array.

func (*AnyValueBuilder) Release 

func (b *AnyValueBuilder) Release()

Release releases the memory allocated by the builder.

type AnyValueStats 

type AnyValueStats struct {
	TotalCount int64
	Missing    int64

	// Attribute type distribution
	I64TypeDistribution    *hdrhistogram.Histogram
	F64TypeDistribution    *hdrhistogram.Histogram
	BoolTypeDistribution   *hdrhistogram.Histogram
	StringTypeDistribution *hdrhistogram.Histogram
	BinaryTypeDistribution *hdrhistogram.Histogram
	ListTypeDistribution   *hdrhistogram.Histogram
	MapTypeDistribution    *hdrhistogram.Histogram

	// Attribute distinct values per type
	I64DistinctValue    *hyperloglog.Sketch
	F64DistinctValue    *hyperloglog.Sketch
	StringDistinctValue *hyperloglog.Sketch
	BinaryDistinctValue *hyperloglog.Sketch

	// Content length distribution
	StringLenDistribution *hdrhistogram.Histogram
	BinaryLenDistribution *hdrhistogram.Histogram
}

func NewAnyValueStats 

func NewAnyValueStats() *AnyValueStats

func (*AnyValueStats) IsPresent 

func (a *AnyValueStats) IsPresent() bool

func (*AnyValueStats) ShowStats 

func (a *AnyValueStats) ShowStats(indent string, title string, color string)

func (*AnyValueStats) UpdateWith 

func (a *AnyValueStats) UpdateWith(value pcommon.Value)

type Attr16 

type Attr16 struct {
	ParentID uint16
	Key      string
	Value    *pcommon.Value
}

Attr16 is an attribute with a 16-bit ParentID.

type Attr32 

type Attr32 struct {
	ParentID uint32
	Key      string
	Value    *pcommon.Value
}

Attr32 is an attribute with a 32-bit ParentID.

type AttrColumn 

type AttrColumn interface {
	ColName() string
	ColType() arrow.DataType
	ColMetadata() arrow.Metadata
	Append(v pcommon.Value)
	AppendNull()
	Len() int
	SetBuilder(builder array.Builder)
	Build() error
	Compare(i, j int) int
	Reset()
}

type Attributes16Accumulator 

type Attributes16Accumulator struct {
	// contains filtered or unexported fields
}

Attributes16Accumulator accumulates attributes for the scope of an entire batch. It is used to sort globally all attributes and optimize the compression ratio. Attribute IDs are 16-bit.

func NewAttributes16Accumulator 

func NewAttributes16Accumulator(sorter Attrs16Sorter) *Attributes16Accumulator

func (*Attributes16Accumulator) Append 

func (c *Attributes16Accumulator) Append(attrs pcommon.Map) (int64, error)

func (*Attributes16Accumulator) AppendWithID 

func (c *Attributes16Accumulator) AppendWithID(parentID uint16, attrs pcommon.Map) error

func (*Attributes16Accumulator) IsEmpty 

func (c *Attributes16Accumulator) IsEmpty() bool

func (*Attributes16Accumulator) Reset 

func (c *Attributes16Accumulator) Reset()

func (*Attributes16Accumulator) Sort 

func (c *Attributes16Accumulator) Sort()

Sort sorts the attributes based on the provided sorter. The sorter is part of the global configuration and can be different for different payload types.

type Attributes32Accumulator 

type Attributes32Accumulator struct {
	// contains filtered or unexported fields
}

Attributes32Accumulator accumulates attributes for the scope of an entire batch. It is used to sort globally all attributes and optimize the compression ratio. Attribute IDs are 32-bit.

func NewAttributes32Accumulator 

func NewAttributes32Accumulator(sorter Attrs32Sorter) *Attributes32Accumulator

func (*Attributes32Accumulator) Append 

func (c *Attributes32Accumulator) Append(ID uint32, attrs pcommon.Map) error

func (*Attributes32Accumulator) IsEmpty 

func (c *Attributes32Accumulator) IsEmpty() bool

func (*Attributes32Accumulator) Reset 

func (c *Attributes32Accumulator) Reset()

func (*Attributes32Accumulator) Sort 

func (c *Attributes32Accumulator) Sort()

Sort sorts the attributes based on the provided sorter. The sorter is part of the global configuration and can be different for different payload types.

type AttributesBuilder 

type AttributesBuilder struct {
	// contains filtered or unexported fields
}

AttributesBuilder is a helper to build a map of attributes.

func AttributesBuilderFrom 

func AttributesBuilderFrom(mb *builder.MapBuilder) *AttributesBuilder

AttributesBuilderFrom creates a new AttributesBuilder from an existing MapBuilder.

func NewAttributesBuilder 

func NewAttributesBuilder(builder *builder.MapBuilder) *AttributesBuilder

NewAttributesBuilder creates a new AttributesBuilder with a given allocator.

Once the builder is no longer needed, Build() or Release() must be called to free the memory allocated by the builder.

func (*AttributesBuilder) Append 

func (b *AttributesBuilder) Append(attrs pcommon.Map) error

Append appends a new set of attributes to the builder. Note: empty keys are skipped.

func (*AttributesBuilder) AppendNull 

func (b *AttributesBuilder) AppendNull() error

func (*AttributesBuilder) Build 

func (b *AttributesBuilder) Build() (*array.Map, error)

Build builds the attribute array map.

Once the returned array is no longer needed, Release() must be called to free the memory allocated by the array.

func (*AttributesBuilder) Release 

func (b *AttributesBuilder) Release()

Release releases the memory allocated by the builder.

type AttributesStats 

type AttributesStats struct {
	TotalCount   int64
	Missing      int64
	Distribution *hdrhistogram.Histogram

	// Attribute type distribution
	I64TypeDistribution    *hdrhistogram.Histogram
	F64TypeDistribution    *hdrhistogram.Histogram
	BoolTypeDistribution   *hdrhistogram.Histogram
	StringTypeDistribution *hdrhistogram.Histogram
	BinaryTypeDistribution *hdrhistogram.Histogram
	ListTypeDistribution   *hdrhistogram.Histogram
	MapTypeDistribution    *hdrhistogram.Histogram

	// Attribute key
	KeyLenDistribution *hdrhistogram.Histogram
	KeyDistinctValue   *hyperloglog.Sketch

	// Attribute distinct values per type
	I64DistinctValue    *hyperloglog.Sketch
	F64DistinctValue    *hyperloglog.Sketch
	StringDistinctValue *hyperloglog.Sketch
	BinaryDistinctValue *hyperloglog.Sketch

	// Content length distribution
	StringLenDistribution *hdrhistogram.Histogram
	BinaryLenDistribution *hdrhistogram.Histogram

	// Dropped Attributes Count distribution
	DACDistinctValue *hyperloglog.Sketch
}

func NewAttributesStats 

func NewAttributesStats() *AttributesStats

func (*AttributesStats) IsPresent 

func (a *AttributesStats) IsPresent() bool

func (*AttributesStats) ShowStats 

func (a *AttributesStats) ShowStats(indent string, title string, color string)

func (*AttributesStats) UpdateWith 

func (a *AttributesStats) UpdateWith(attrs pcommon.Map, dac uint32)

type Attrs16Builder 

type Attrs16Builder struct {
	// contains filtered or unexported fields
}

func NewAttrs16BuilderWithEncoding 

func NewAttrs16BuilderWithEncoding(rBuilder *builder.RecordBuilderExt, payloadType *PayloadType, config *Attrs16Config) *Attrs16Builder

func (*Attrs16Builder) Accumulator 

func (b *Attrs16Builder) Accumulator() *Attributes16Accumulator

func (*Attrs16Builder) Build 

func (b *Attrs16Builder) Build() (arrow.Record, error)

func (*Attrs16Builder) IsEmpty 

func (b *Attrs16Builder) IsEmpty() bool

func (*Attrs16Builder) PayloadType 

func (b *Attrs16Builder) PayloadType() *PayloadType

func (*Attrs16Builder) Release 

func (b *Attrs16Builder) Release()

Release releases the memory allocated by the builder.

func (*Attrs16Builder) Reset 

func (b *Attrs16Builder) Reset()

func (*Attrs16Builder) Schema 

func (b *Attrs16Builder) Schema() *arrow.Schema

func (*Attrs16Builder) SchemaID 

func (b *Attrs16Builder) SchemaID() string

func (*Attrs16Builder) ShowSchema 

func (b *Attrs16Builder) ShowSchema()

func (*Attrs16Builder) TryBuild 

func (b *Attrs16Builder) TryBuild() (record arrow.Record, err error)

type Attrs16ByKeyParentIdValue 

type Attrs16ByKeyParentIdValue struct {
	// contains filtered or unexported fields
}

func SortAttrs16ByKeyParentIdValue 

func SortAttrs16ByKeyParentIdValue() *Attrs16ByKeyParentIdValue

func (*Attrs16ByKeyParentIdValue) Encode 

func (s *Attrs16ByKeyParentIdValue) Encode(parentID uint16, key string, value *pcommon.Value) uint16

func (*Attrs16ByKeyParentIdValue) IsSameGroup 

func (s *Attrs16ByKeyParentIdValue) IsSameGroup(key string, value *pcommon.Value) bool

func (*Attrs16ByKeyParentIdValue) Reset 

func (s *Attrs16ByKeyParentIdValue) Reset()

func (*Attrs16ByKeyParentIdValue) Sort 

func (s *Attrs16ByKeyParentIdValue) Sort(attrs []Attr16)

type Attrs16ByKeyValueParentId 

type Attrs16ByKeyValueParentId struct {
	// contains filtered or unexported fields
}

func SortAttrs16ByKeyValueParentId 

func SortAttrs16ByKeyValueParentId() *Attrs16ByKeyValueParentId

func (*Attrs16ByKeyValueParentId) Encode 

func (s *Attrs16ByKeyValueParentId) Encode(parentID uint16, key string, value *pcommon.Value) uint16

func (*Attrs16ByKeyValueParentId) IsSameGroup 

func (s *Attrs16ByKeyValueParentId) IsSameGroup(key string, value *pcommon.Value) bool

func (*Attrs16ByKeyValueParentId) Reset 

func (s *Attrs16ByKeyValueParentId) Reset()

func (*Attrs16ByKeyValueParentId) Sort 

func (s *Attrs16ByKeyValueParentId) Sort(attrs []Attr16)

type Attrs16ByNothing 

type Attrs16ByNothing struct{}

func UnsortedAttrs16 

func UnsortedAttrs16() *Attrs16ByNothing

func (*Attrs16ByNothing) Encode 

func (s *Attrs16ByNothing) Encode(parentID uint16, _ string, _ *pcommon.Value) uint16

func (*Attrs16ByNothing) Reset 

func (s *Attrs16ByNothing) Reset()

func (*Attrs16ByNothing) Sort 

func (s *Attrs16ByNothing) Sort(_ []Attr16)

type Attrs16ByParentIdKeyValue 

type Attrs16ByParentIdKeyValue struct {
	// contains filtered or unexported fields
}

func SortByParentIdKeyValueAttr16 

func SortByParentIdKeyValueAttr16() *Attrs16ByParentIdKeyValue

func (*Attrs16ByParentIdKeyValue) Encode 

func (s *Attrs16ByParentIdKeyValue) Encode(parentID uint16, _ string, _ *pcommon.Value) uint16

func (*Attrs16ByParentIdKeyValue) Reset 

func (s *Attrs16ByParentIdKeyValue) Reset()

func (*Attrs16ByParentIdKeyValue) Sort 

func (s *Attrs16ByParentIdKeyValue) Sort(attrs []Attr16)

type Attrs16Config 

type Attrs16Config struct {
	Sorter Attrs16Sorter
}

type Attrs16Sorter 

type Attrs16Sorter interface {
	Sort(attrs []Attr16)
	Encode(parentID uint16, key string, value *pcommon.Value) uint16
	Reset()
}

Attrs16Sorter is used to sort attributes with 16-bit ParentIDs.

type Attrs32Builder 

type Attrs32Builder struct {
	// contains filtered or unexported fields
}

func NewAttrs32Builder 

func NewAttrs32Builder(rBuilder *builder.RecordBuilderExt, payloadType *PayloadType, sorter Attrs32Sorter) *Attrs32Builder

func NewAttrs32BuilderWithEncoding 

func NewAttrs32BuilderWithEncoding(rBuilder *builder.RecordBuilderExt, payloadType *PayloadType, conf *Attrs32Config) *Attrs32Builder

func (*Attrs32Builder) Accumulator 

func (b *Attrs32Builder) Accumulator() *Attributes32Accumulator

func (*Attrs32Builder) Build 

func (b *Attrs32Builder) Build() (arrow.Record, error)

func (*Attrs32Builder) IsEmpty 

func (b *Attrs32Builder) IsEmpty() bool

func (*Attrs32Builder) PayloadType 

func (b *Attrs32Builder) PayloadType() *PayloadType

func (*Attrs32Builder) Release 

func (b *Attrs32Builder) Release()

Release releases the memory allocated by the builder.

func (*Attrs32Builder) Reset 

func (b *Attrs32Builder) Reset()

func (*Attrs32Builder) Schema 

func (b *Attrs32Builder) Schema() *arrow.Schema

func (*Attrs32Builder) SchemaID 

func (b *Attrs32Builder) SchemaID() string

func (*Attrs32Builder) ShowSchema 

func (b *Attrs32Builder) ShowSchema()

func (*Attrs32Builder) TryBuild 

func (b *Attrs32Builder) TryBuild() (record arrow.Record, err error)

type Attrs32ByKeyParentIdValue 

type Attrs32ByKeyParentIdValue struct {
	// contains filtered or unexported fields
}

func SortAttrs32ByKeyParentIdValue 

func SortAttrs32ByKeyParentIdValue() *Attrs32ByKeyParentIdValue

func (*Attrs32ByKeyParentIdValue) Encode 

func (s *Attrs32ByKeyParentIdValue) Encode(parentID uint32, key string, value *pcommon.Value) uint32

func (*Attrs32ByKeyParentIdValue) IsSameGroup 

func (s *Attrs32ByKeyParentIdValue) IsSameGroup(key string, value *pcommon.Value) bool

func (*Attrs32ByKeyParentIdValue) Reset 

func (s *Attrs32ByKeyParentIdValue) Reset()

func (*Attrs32ByKeyParentIdValue) Sort 

func (s *Attrs32ByKeyParentIdValue) Sort(attrs []Attr32)

type Attrs32ByKeyValueParentId 

type Attrs32ByKeyValueParentId struct {
	// contains filtered or unexported fields
}

func SortAttrs32ByKeyValueParentId 

func SortAttrs32ByKeyValueParentId() *Attrs32ByKeyValueParentId

func (*Attrs32ByKeyValueParentId) Encode 

func (s *Attrs32ByKeyValueParentId) Encode(parentID uint32, key string, value *pcommon.Value) uint32

func (*Attrs32ByKeyValueParentId) IsSameGroup 

func (s *Attrs32ByKeyValueParentId) IsSameGroup(key string, value *pcommon.Value) bool

func (*Attrs32ByKeyValueParentId) Reset 

func (s *Attrs32ByKeyValueParentId) Reset()

func (*Attrs32ByKeyValueParentId) Sort 

func (s *Attrs32ByKeyValueParentId) Sort(attrs []Attr32)

type Attrs32ByNothing 

type Attrs32ByNothing struct{}

func UnsortedAttrs32 

func UnsortedAttrs32() *Attrs32ByNothing

func (*Attrs32ByNothing) Encode 

func (s *Attrs32ByNothing) Encode(parentID uint32, _ string, _ *pcommon.Value) uint32

func (*Attrs32ByNothing) Reset 

func (s *Attrs32ByNothing) Reset()

func (*Attrs32ByNothing) Sort 

func (s *Attrs32ByNothing) Sort(_ []Attr32)

type Attrs32ByTypeParentIdKeyValue 

type Attrs32ByTypeParentIdKeyValue struct {
	// contains filtered or unexported fields
}

func SortAttrs32ByTypeParentIdKeyValue 

func SortAttrs32ByTypeParentIdKeyValue() *Attrs32ByTypeParentIdKeyValue

func (*Attrs32ByTypeParentIdKeyValue) Encode 

func (s *Attrs32ByTypeParentIdKeyValue) Encode(parentID uint32, _ string, value *pcommon.Value) uint32

func (*Attrs32ByTypeParentIdKeyValue) Reset 

func (s *Attrs32ByTypeParentIdKeyValue) Reset()

func (*Attrs32ByTypeParentIdKeyValue) Sort 

func (s *Attrs32ByTypeParentIdKeyValue) Sort(attrs []Attr32)

type Attrs32Config 

type Attrs32Config struct {
	Sorter Attrs32Sorter
}

type Attrs32Sorter 

type Attrs32Sorter interface {
	Sort(attrs []Attr32)
	Encode(parentID uint32, key string, value *pcommon.Value) uint32
	Reset()
}

Attrs32Sorter is used to sort attributes with 32-bit ParentIDs.

type BinaryAttrColumn 

type BinaryAttrColumn struct {
	// contains filtered or unexported fields
}

func (*BinaryAttrColumn) Append 

func (c *BinaryAttrColumn) Append(v pcommon.Value)

func (*BinaryAttrColumn) AppendNull 

func (c *BinaryAttrColumn) AppendNull()

func (*BinaryAttrColumn) Build 

func (c *BinaryAttrColumn) Build() error

func (*BinaryAttrColumn) ColMetadata 

func (c *BinaryAttrColumn) ColMetadata() arrow.Metadata

func (*BinaryAttrColumn) ColName 

func (c *BinaryAttrColumn) ColName() string

func (*BinaryAttrColumn) ColType 

func (c *BinaryAttrColumn) ColType() arrow.DataType

func (*BinaryAttrColumn) Compare 

func (c *BinaryAttrColumn) Compare(i, j int) int

func (*BinaryAttrColumn) Len 

func (c *BinaryAttrColumn) Len() int

func (*BinaryAttrColumn) Reset 

func (c *BinaryAttrColumn) Reset()

func (*BinaryAttrColumn) SetBuilder 

func (c *BinaryAttrColumn) SetBuilder(builder array.Builder)

type BoolAttrColumn 

type BoolAttrColumn struct {
	// contains filtered or unexported fields
}

func (*BoolAttrColumn) Append 

func (c *BoolAttrColumn) Append(v pcommon.Value)

func (*BoolAttrColumn) AppendNull 

func (c *BoolAttrColumn) AppendNull()

func (*BoolAttrColumn) Build 

func (c *BoolAttrColumn) Build() error

func (*BoolAttrColumn) ColMetadata 

func (c *BoolAttrColumn) ColMetadata() arrow.Metadata

func (*BoolAttrColumn) ColName 

func (c *BoolAttrColumn) ColName() string

func (*BoolAttrColumn) ColType 

func (c *BoolAttrColumn) ColType() arrow.DataType

func (*BoolAttrColumn) Compare 

func (c *BoolAttrColumn) Compare(i, j int) int

func (*BoolAttrColumn) Len 

func (c *BoolAttrColumn) Len() int

func (*BoolAttrColumn) Reset 

func (c *BoolAttrColumn) Reset()

func (*BoolAttrColumn) SetBuilder 

func (c *BoolAttrColumn) SetBuilder(builder array.Builder)

type CborAttrColumn 

type CborAttrColumn struct {
	// contains filtered or unexported fields
}

func (*CborAttrColumn) Append 

func (c *CborAttrColumn) Append(v pcommon.Value)

func (*CborAttrColumn) AppendNull 

func (c *CborAttrColumn) AppendNull()

func (*CborAttrColumn) Build 

func (c *CborAttrColumn) Build() error

func (*CborAttrColumn) ColMetadata 

func (c *CborAttrColumn) ColMetadata() arrow.Metadata

func (*CborAttrColumn) ColName 

func (c *CborAttrColumn) ColName() string

func (*CborAttrColumn) ColType 

func (c *CborAttrColumn) ColType() arrow.DataType

func (*CborAttrColumn) Compare 

func (c *CborAttrColumn) Compare(i, j int) int

func (*CborAttrColumn) Len 

func (c *CborAttrColumn) Len() int

func (*CborAttrColumn) Reset 

func (c *CborAttrColumn) Reset()

func (*CborAttrColumn) SetBuilder 

func (c *CborAttrColumn) SetBuilder(builder array.Builder)

type DoubleAttrColumn 

type DoubleAttrColumn struct {
	// contains filtered or unexported fields
}

func (*DoubleAttrColumn) Append 

func (c *DoubleAttrColumn) Append(v pcommon.Value)

func (*DoubleAttrColumn) AppendNull 

func (c *DoubleAttrColumn) AppendNull()

func (*DoubleAttrColumn) Build 

func (c *DoubleAttrColumn) Build() error

func (*DoubleAttrColumn) ColMetadata 

func (c *DoubleAttrColumn) ColMetadata() arrow.Metadata

func (*DoubleAttrColumn) ColName 

func (c *DoubleAttrColumn) ColName() string

func (*DoubleAttrColumn) ColType 

func (c *DoubleAttrColumn) ColType() arrow.DataType

func (*DoubleAttrColumn) Compare 

func (c *DoubleAttrColumn) Compare(i, j int) int

func (*DoubleAttrColumn) Len 

func (c *DoubleAttrColumn) Len() int

func (*DoubleAttrColumn) Reset 

func (c *DoubleAttrColumn) Reset()

func (*DoubleAttrColumn) SetBuilder 

func (c *DoubleAttrColumn) SetBuilder(builder array.Builder)

type DynAttrsBuilder 

type DynAttrsBuilder struct {
	// contains filtered or unexported fields
}

func NewDynAttrsBuilder 

func NewDynAttrsBuilder(payloadType *PayloadType, mem memory.Allocator) *DynAttrsBuilder

func (*DynAttrsBuilder) Append 

func (b *DynAttrsBuilder) Append(parentID uint32, attrs pcommon.Map) error

func (*DynAttrsBuilder) Build 

func (b *DynAttrsBuilder) Build() (arrow.Record, error)

func (*DynAttrsBuilder) Compare 

func (b *DynAttrsBuilder) Compare(rowI, rowJ, colIdx int) int

func (*DynAttrsBuilder) IsEmpty 

func (b *DynAttrsBuilder) IsEmpty() bool

func (*DynAttrsBuilder) PayloadType 

func (b *DynAttrsBuilder) PayloadType() *PayloadType

func (*DynAttrsBuilder) Release 

func (b *DynAttrsBuilder) Release()

Release releases the memory allocated by the builder.

func (*DynAttrsBuilder) Reset 

func (b *DynAttrsBuilder) Reset()

func (*DynAttrsBuilder) Schema 

func (b *DynAttrsBuilder) Schema() *arrow.Schema

func (*DynAttrsBuilder) SchemaID 

func (b *DynAttrsBuilder) SchemaID() string

func (*DynAttrsBuilder) SchemaUpdateCount 

func (b *DynAttrsBuilder) SchemaUpdateCount() int

type EntityBuilder 

type EntityBuilder[T pmetric.Metrics | plog.Logs | ptrace.Traces] interface {
	Append(T) error
	Build() (arrow.Record, error)
}

type IntAttrColumn 

type IntAttrColumn struct {
	// contains filtered or unexported fields
}

func (*IntAttrColumn) Append 

func (c *IntAttrColumn) Append(v pcommon.Value)

func (*IntAttrColumn) AppendNull 

func (c *IntAttrColumn) AppendNull()

func (*IntAttrColumn) Build 

func (c *IntAttrColumn) Build() error

func (*IntAttrColumn) ColMetadata 

func (c *IntAttrColumn) ColMetadata() arrow.Metadata

func (*IntAttrColumn) ColName 

func (c *IntAttrColumn) ColName() string

func (*IntAttrColumn) ColType 

func (c *IntAttrColumn) ColType() arrow.DataType

func (*IntAttrColumn) Compare 

func (c *IntAttrColumn) Compare(i, j int) int

func (*IntAttrColumn) Len 

func (c *IntAttrColumn) Len() int

func (*IntAttrColumn) Reset 

func (c *IntAttrColumn) Reset()

func (*IntAttrColumn) SetBuilder 

func (c *IntAttrColumn) SetBuilder(builder array.Builder)

type LimitError 

type LimitError struct {
	Request uint64
	Inuse   uint64
	Limit   uint64
}

func (LimitError) Error 

func (le LimitError) Error() string

func (LimitError) Is 

func (_ LimitError) Is(tgt error) bool

type LimitedAllocator 

type LimitedAllocator struct {
	// contains filtered or unexported fields
}

func NewLimitedAllocator 

func NewLimitedAllocator(mem memory.Allocator, limit uint64) *LimitedAllocator

func (*LimitedAllocator) Allocate 

func (l *LimitedAllocator) Allocate(size int) []byte

func (*LimitedAllocator) Free 

func (l *LimitedAllocator) Free(b []byte)

func (*LimitedAllocator) Reallocate 

func (l *LimitedAllocator) Reallocate(size int, b []byte) []byte

type Options 

type Options struct {
	Sort  bool
	Stats bool
}

type ParentIDColumn 

type ParentIDColumn struct {
	// contains filtered or unexported fields
}

func (*ParentIDColumn) Build 

func (c *ParentIDColumn) Build()

func (*ParentIDColumn) Reset 

func (c *ParentIDColumn) Reset()

type PayloadType 

type PayloadType struct {
	// contains filtered or unexported fields
}

PayloadType wraps the protobuf payload type generated from the protobuf definition and adds a prefix to it.

func (*PayloadType) PayloadType 

func (p *PayloadType) PayloadType() record_message.PayloadType

func (*PayloadType) SchemaPrefix 

func (p *PayloadType) SchemaPrefix() string

type RelatedRecordBuilder 

type RelatedRecordBuilder interface {
	IsEmpty() bool
	Build() (arrow.Record, error)
	SchemaID() string
	Schema() *arrow.Schema
	PayloadType() *PayloadType
	Reset()
	Release()
}

RelatedRecordBuilder is the common interface for all related record builders.

type RelatedRecordsManager 

type RelatedRecordsManager struct {
	// contains filtered or unexported fields
}

RelatedRecordsManager manages all related record builders for a given main OTel entity.

func NewRelatedRecordsManager 

func NewRelatedRecordsManager(cfg *cfg.Config, stats *stats.ProducerStats) *RelatedRecordsManager

func (*RelatedRecordsManager) BuildRecordMessages 

func (m *RelatedRecordsManager) BuildRecordMessages() ([]*record_message.RecordMessage, error)

func (*RelatedRecordsManager) Declare 

func (m *RelatedRecordsManager) Declare(payloadType *PayloadType, parentPayloadType *PayloadType, schema *arrow.Schema, rrBuilder func(b *builder.RecordBuilderExt) RelatedRecordBuilder) RelatedRecordBuilder

func (*RelatedRecordsManager) RecordBuilderExt 

func (m *RelatedRecordsManager) RecordBuilderExt(payloadType *PayloadType) *builder.RecordBuilderExt

func (*RelatedRecordsManager) Release 

func (m *RelatedRecordsManager) Release()

func (*RelatedRecordsManager) Reset 

func (m *RelatedRecordsManager) Reset()

func (*RelatedRecordsManager) Schemas 

func (m *RelatedRecordsManager) Schemas() []SchemaWithPayload

type ResourceBuilder 

type ResourceBuilder struct {
	// contains filtered or unexported fields
}

ResourceBuilder is an Arrow builder for resources.

func ResourceBuilderFrom 

func ResourceBuilderFrom(builder *builder.StructBuilder) *ResourceBuilder

ResourceBuilderFrom creates a new resource builder from an existing struct builder.

func (*ResourceBuilder) Append 

func (b *ResourceBuilder) Append(resID int64, resource pcommon.Resource, schemaUrl string) error

func (*ResourceBuilder) Build 

func (b *ResourceBuilder) Build() (*array.Struct, error)

Build builds the resource array struct.

Once the array is no longer needed, Release() must be called to free the memory allocated by the array.

func (*ResourceBuilder) Release 

func (b *ResourceBuilder) Release()

Release releases the memory allocated by the builder.

type ResourceStats 

type ResourceStats struct {
	TotalCount int64
	Missing    int64

	AttributesStats *AttributesStats
}

func (*ResourceStats) ShowStats 

func (r *ResourceStats) ShowStats(indent string)

func (*ResourceStats) UpdateWith 

func (r *ResourceStats) UpdateWith(res pcommon.Resource)

type SchemaUrlStats 

type SchemaUrlStats struct {
	Missing          int64
	NonEmpty         int64
	SizeDistribution *hdrhistogram.Histogram
}

func (*SchemaUrlStats) ShowStats 

func (s *SchemaUrlStats) ShowStats(indent string)

func (*SchemaUrlStats) UpdateWith 

func (s *SchemaUrlStats) UpdateWith(schemaUrl string)

type SchemaWithPayload 

type SchemaWithPayload struct {
	Schema            *arrow.Schema
	PayloadType       *PayloadType
	ParentPayloadType *PayloadType
}

type ScopeBuilder 

type ScopeBuilder struct {
	// contains filtered or unexported fields
}

func ScopeBuilderFrom 

func ScopeBuilderFrom(sb *builder.StructBuilder) *ScopeBuilder

ScopeBuilderFrom creates a new instrumentation scope array builder from an existing struct builder.

func (*ScopeBuilder) Append 

func (b *ScopeBuilder) Append(scopeID int64, scope pcommon.InstrumentationScope) error

func (*ScopeBuilder) Build 

func (b *ScopeBuilder) Build() (*array.Struct, error)

Build builds the instrumentation scope array struct.

Once the array is no longer needed, Release() must be called to free the memory allocated by the array.

func (*ScopeBuilder) Release 

func (b *ScopeBuilder) Release()

Release releases the memory allocated by the builder.

type ScopeStats 

type ScopeStats struct {
	TotalCount int64
	Missing    int64

	AttributesStats *AttributesStats

	Name    *StringStats
	Version *StringStats
}

func (*ScopeStats) ShowStats 

func (s *ScopeStats) ShowStats(indent string)

func (*ScopeStats) UpdateWith 

func (s *ScopeStats) UpdateWith(scope pcommon.InstrumentationScope)

type StatusStats 

type StatusStats struct {
	TotalCount             int64
	Missing                int64
	CodeDistinctValue      *hyperloglog.Sketch
	MissingCode            int64
	MessageDistincValue    *hyperloglog.Sketch
	MessageLenDistribution *hdrhistogram.Histogram
	MissingMessage         int64
}

func NewStatusStats 

func NewStatusStats() *StatusStats

func (*StatusStats) ShowStats 

func (s *StatusStats) ShowStats(indent string)

func (*StatusStats) UpdateWith 

func (s *StatusStats) UpdateWith(status ptrace.Status)

type StringAttrColumn 

type StringAttrColumn struct {
	// contains filtered or unexported fields
}

func (*StringAttrColumn) Append 

func (c *StringAttrColumn) Append(v pcommon.Value)

func (*StringAttrColumn) AppendNull 

func (c *StringAttrColumn) AppendNull()

func (*StringAttrColumn) Build 

func (c *StringAttrColumn) Build() error

func (*StringAttrColumn) ColMetadata 

func (c *StringAttrColumn) ColMetadata() arrow.Metadata

func (*StringAttrColumn) ColName 

func (c *StringAttrColumn) ColName() string

func (*StringAttrColumn) ColType 

func (c *StringAttrColumn) ColType() arrow.DataType

func (*StringAttrColumn) Compare 

func (c *StringAttrColumn) Compare(i, j int) int

func (*StringAttrColumn) Len 

func (c *StringAttrColumn) Len() int

func (*StringAttrColumn) Reset 

func (c *StringAttrColumn) Reset()

func (*StringAttrColumn) SetBuilder 

func (c *StringAttrColumn) SetBuilder(builder array.Builder)

type StringStats 

type StringStats struct {
	Missing         int64
	LenDistribution *hdrhistogram.Histogram
	DistinctValue   *hyperloglog.Sketch
}

func NewStringStats 

func NewStringStats() *StringStats

func (*StringStats) IsPresent 

func (s *StringStats) IsPresent() bool

func (*StringStats) ShowStats 

func (s *StringStats) ShowStats(name string, indent string)

func (*StringStats) UpdateWith 

func (s *StringStats) UpdateWith(str string)

type TimeIntervalStats 

type TimeIntervalStats struct {
	TotalCount              int64
	StartTimeDistinctValue  *hyperloglog.Sketch
	EndTimeDistinctValue    *hyperloglog.Sketch
	IntervalDistinctValue   *hyperloglog.Sketch
	StartDeltaDistinctValue *hyperloglog.Sketch
	EndDeltaDistinctValue   *hyperloglog.Sketch
}

func NewTimeIntervalStats 

func NewTimeIntervalStats() *TimeIntervalStats

func (*TimeIntervalStats) ShowStats 

func (t *TimeIntervalStats) ShowStats(indent string)

func (*TimeIntervalStats) UpdateWithSpans 

func (t *TimeIntervalStats) UpdateWithSpans(spans []ptrace.Span)

type TimestampStats 

type TimestampStats struct {
	TotalCount        int64
	TimeDistinctValue *hyperloglog.Sketch
}

func NewTimestampStats 

func NewTimestampStats() *TimestampStats

func (*TimestampStats) ShowStats 

func (t *TimestampStats) ShowStats(title, indent string)

func (*TimestampStats) UpdateWith 

func (t *TimestampStats) UpdateWith(timestamp pcommon.Timestamp)

type ValueTypeCounters 

type ValueTypeCounters struct {
	// contains filtered or unexported fields
}

ValueTypeCounters is a struct to count the number of values of each type.

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.