arrow

package
v0.0.0-...-cafc5b9 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Aug 16, 2023 License: Apache-2.0 Imports: 21 Imported by: 0

Documentation

Overview

Package arrow contains types and functions used to convert OTLP traces into their Arrow representation.

Index

Constants

View Source
const None = ""

Variables

View Source
var (
	// EventSchema is the Arrow schema representing events.
	// Related record.
	EventSchema = arrow.NewSchema([]arrow.Field{
		{Name: constants.ID, Type: arrow.PrimitiveTypes.Uint32, Metadata: schema.Metadata(schema.DeltaEncoding), Nullable: true},
		{Name: constants.ParentID, Type: arrow.PrimitiveTypes.Uint16},
		{Name: constants.TimeUnixNano, Type: arrow.FixedWidthTypes.Timestamp_ns, Nullable: true},
		{Name: constants.Name, Type: arrow.BinaryTypes.String, Metadata: schema.Metadata(schema.Dictionary8)},
		{Name: constants.DroppedAttributesCount, Type: arrow.PrimitiveTypes.Uint32, Nullable: true},
	}, nil)
)
View Source
var (
	// LinkSchema is the Arrow Data Type describing a link (as a related record
	// to the main trace record).
	LinkSchema = arrow.NewSchema([]arrow.Field{
		{Name: constants.ID, Type: arrow.PrimitiveTypes.Uint32, Metadata: schema.Metadata(schema.DeltaEncoding), Nullable: true},
		{Name: constants.ParentID, Type: arrow.PrimitiveTypes.Uint16},
		{Name: constants.TraceId, Type: &arrow.FixedSizeBinaryType{ByteWidth: 16}, Metadata: schema.Metadata(schema.Dictionary8), Nullable: true},
		{Name: constants.SpanId, Type: &arrow.FixedSizeBinaryType{ByteWidth: 8}, Metadata: schema.Metadata(schema.Dictionary8), Nullable: true},
		{Name: constants.TraceState, Type: arrow.BinaryTypes.String, Metadata: schema.Metadata(schema.Dictionary8), Nullable: true},
		{Name: constants.DroppedAttributesCount, Type: arrow.PrimitiveTypes.Uint32, Nullable: true},
	}, nil)
)
View Source
var (
	StatusDT = arrow.StructOf([]arrow.Field{
		{Name: constants.StatusCode, Type: arrow.PrimitiveTypes.Int32, Metadata: schema.Metadata(schema.Dictionary8), Nullable: true},
		{Name: constants.StatusMessage, Type: arrow.BinaryTypes.String, Metadata: schema.Metadata(schema.Dictionary8), Nullable: true},
	}...)
)

StatusDT is the Arrow Data Type describing a span status.

View Source
var (
	// TracesSchema is the Arrow schema for the OTLP Arrow Traces record.
	TracesSchema = arrow.NewSchema([]arrow.Field{
		{Name: constants.ID, Type: arrow.PrimitiveTypes.Uint16, Metadata: schema.Metadata(schema.DeltaEncoding), Nullable: true},
		{Name: constants.Resource, Type: acommon.ResourceDT, Nullable: true},
		{Name: constants.Scope, Type: acommon.ScopeDT, Nullable: true},

		{Name: constants.SchemaUrl, Type: arrow.BinaryTypes.String, Metadata: schema.Metadata(schema.Dictionary8), Nullable: true},
		{Name: constants.StartTimeUnixNano, Type: arrow.FixedWidthTypes.Timestamp_ns},
		{Name: constants.DurationTimeUnixNano, Type: arrow.FixedWidthTypes.Duration_ms, Metadata: schema.Metadata(schema.Dictionary8)},
		{Name: constants.TraceId, Type: &arrow.FixedSizeBinaryType{ByteWidth: 16}},
		{Name: constants.SpanId, Type: &arrow.FixedSizeBinaryType{ByteWidth: 8}},
		{Name: constants.TraceState, Type: arrow.BinaryTypes.String, Metadata: schema.Metadata(schema.Dictionary8), Nullable: true},
		{Name: constants.ParentSpanId, Type: &arrow.FixedSizeBinaryType{ByteWidth: 8}, Nullable: true},
		{Name: constants.Name, Type: arrow.BinaryTypes.String, Metadata: schema.Metadata(schema.Dictionary8)},
		{Name: constants.KIND, Type: arrow.PrimitiveTypes.Int32, Metadata: schema.Metadata(schema.Dictionary8), Nullable: true},
		{Name: constants.DroppedAttributesCount, Type: arrow.PrimitiveTypes.Uint32, Nullable: true},
		{Name: constants.DroppedEventsCount, Type: arrow.PrimitiveTypes.Uint32, Nullable: true},
		{Name: constants.DroppedLinksCount, Type: arrow.PrimitiveTypes.Uint32, Nullable: true},
		{Name: constants.Status, Type: StatusDT, Nullable: true},
	}, nil)
)

Functions

This section is empty.

Types

type AttrsBuilders

type AttrsBuilders struct {
	// contains filtered or unexported fields
}

AttrsBuilders groups together AttrsBuilder instances used to build related data attributes (i.e. resource attributes, scope attributes, span attributes, event attributes, and link attributes).

func (*AttrsBuilders) Event

func (ab *AttrsBuilders) Event() *carrow.Attrs32Builder
func (ab *AttrsBuilders) Link() *carrow.Attrs32Builder

func (*AttrsBuilders) Resource

func (ab *AttrsBuilders) Resource() *carrow.Attrs16Builder

func (*AttrsBuilders) Scope

func (ab *AttrsBuilders) Scope() *carrow.Attrs16Builder

func (*AttrsBuilders) Span

func (ab *AttrsBuilders) Span() *carrow.Attrs16Builder

type AttrsConfig

type AttrsConfig struct {
	Resource *arrow.Attrs16Config
	Scope    *arrow.Attrs16Config
	Span     *arrow.Attrs16Config
	Event    *arrow.Attrs32Config
	Link     *arrow.Attrs32Config
}

type Config

type Config struct {
	Global *cfg.Config

	Span  *SpanConfig
	Event *EventConfig
	Link  *LinkConfig
	Attrs *AttrsConfig
}

func DefaultConfig

func DefaultConfig() *Config

func NewConfig

func NewConfig(globalConf *cfg.Config) *Config

func NewNoSortConfig

func NewNoSortConfig(globalConf *cfg.Config) *Config

type Event

type Event struct {
	ParentID               uint16
	TimeUnixNano           pcommon.Timestamp
	Name                   string
	Attributes             pcommon.Map
	DroppedAttributesCount uint32
}

Event is an internal representation of an event used by the EventAccumulator.

type EventAccumulator

type EventAccumulator struct {
	// contains filtered or unexported fields
}

EventAccumulator is an accumulator for events that is used to sort events globally in order to improve compression.

func NewEventAccumulator

func NewEventAccumulator(sorter EventSorter) *EventAccumulator

NewEventAccumulator creates a new EventAccumulator.

func (*EventAccumulator) Append

func (a *EventAccumulator) Append(spanID uint16, events ptrace.SpanEventSlice) error

Append appends a slice of events to the accumulator.

func (*EventAccumulator) IsEmpty

func (a *EventAccumulator) IsEmpty() bool

func (*EventAccumulator) Reset

func (a *EventAccumulator) Reset()

type EventBuilder

type EventBuilder struct {
	// contains filtered or unexported fields
}

EventBuilder is an Arrow builder for events.

func NewEventBuilder

func NewEventBuilder(rBuilder *builder.RecordBuilderExt, conf *EventConfig) *EventBuilder

func (*EventBuilder) Accumulator

func (b *EventBuilder) Accumulator() *EventAccumulator

func (*EventBuilder) Build

func (b *EventBuilder) Build() (record arrow.Record, err error)

func (*EventBuilder) IsEmpty

func (b *EventBuilder) IsEmpty() bool

func (*EventBuilder) PayloadType

func (b *EventBuilder) PayloadType() *acommon.PayloadType

func (*EventBuilder) Release

func (b *EventBuilder) Release()

Release releases the memory allocated by the builder.

func (*EventBuilder) Reset

func (b *EventBuilder) Reset()

func (*EventBuilder) Schema

func (b *EventBuilder) Schema() *arrow.Schema

func (*EventBuilder) SchemaID

func (b *EventBuilder) SchemaID() string

func (*EventBuilder) SetAttributesAccumulator

func (b *EventBuilder) SetAttributesAccumulator(accu *acommon.Attributes32Accumulator)

func (*EventBuilder) TryBuild

func (b *EventBuilder) TryBuild(attrsAccu *acommon.Attributes32Accumulator) (record arrow.Record, err error)

type EventConfig

type EventConfig struct {
	Sorter EventSorter
}

type EventSorter

type EventSorter interface {
	Sort(events []*Event)
	Encode(parentID uint16, event *Event) uint16
	Reset()
}

type EventStats

type EventStats struct {
	TotalCount   int64
	Missing      int64
	Distribution *hdrhistogram.Histogram
	Timestamp    *carrow.TimestampStats
	Name         *carrow.StringStats
	Attributes   *carrow.AttributesStats
}

func NewEventStats

func NewEventStats() *EventStats

func (*EventStats) ShowStats

func (e *EventStats) ShowStats(indent string)

func (*EventStats) UpdateWith

func (e *EventStats) UpdateWith(events ptrace.SpanEventSlice, dac uint32)

type EventsByNameParentId

type EventsByNameParentId struct {
	// contains filtered or unexported fields
}

func SortEventsByNameParentId

func SortEventsByNameParentId() *EventsByNameParentId

func (*EventsByNameParentId) Encode

func (s *EventsByNameParentId) Encode(parentID uint16, event *Event) uint16

func (*EventsByNameParentId) IsSameGroup

func (s *EventsByNameParentId) IsSameGroup(event *Event) bool

func (*EventsByNameParentId) Reset

func (s *EventsByNameParentId) Reset()

func (*EventsByNameParentId) Sort

func (s *EventsByNameParentId) Sort(events []*Event)

type EventsByNameTimeUnixNano

type EventsByNameTimeUnixNano struct {
	// contains filtered or unexported fields
}

func SortEventsByNameTimeUnixNano

func SortEventsByNameTimeUnixNano() *EventsByNameTimeUnixNano

func (*EventsByNameTimeUnixNano) Encode

func (s *EventsByNameTimeUnixNano) Encode(parentID uint16, _ *Event) uint16

func (*EventsByNameTimeUnixNano) Reset

func (s *EventsByNameTimeUnixNano) Reset()

func (*EventsByNameTimeUnixNano) Sort

func (s *EventsByNameTimeUnixNano) Sort(events []*Event)

type EventsByNothing

type EventsByNothing struct{}

func UnsortedEvents

func UnsortedEvents() *EventsByNothing

func (*EventsByNothing) Encode

func (s *EventsByNothing) Encode(parentID uint16, _ *Event) uint16

func (*EventsByNothing) Reset

func (s *EventsByNothing) Reset()

func (*EventsByNothing) Sort

func (s *EventsByNothing) Sort(_ []*Event)

type FlattenedSpan

type FlattenedSpan struct {
	// Resource span section.
	ResourceSpanID    string
	Resource          pcommon.Resource
	ResourceSchemaUrl string

	// Scope span section.
	ScopeSpanID    string
	Scope          pcommon.InstrumentationScope
	ScopeSchemaUrl string

	// Span section.
	Span ptrace.Span
}
type Link struct {
	ParentID               uint16
	TraceID                [16]byte
	SpanID                 [8]byte
	TraceState             string
	Attributes             pcommon.Map
	DroppedAttributesCount uint32
}

Link is an internal representation of a link used by the LinkAccumulator.

type LinkAccumulator

type LinkAccumulator struct {
	// contains filtered or unexported fields
}

LinkAccumulator is an accumulator for links that is used to sort links globally in order to improve compression.

func NewLinkAccumulator

func NewLinkAccumulator(sorter LinkSorter) *LinkAccumulator

NewLinkAccumulator creates a new LinkAccumulator.

func (*LinkAccumulator) Append

func (a *LinkAccumulator) Append(spanID uint16, links ptrace.SpanLinkSlice) error

Append appends a new link to the builder.

func (*LinkAccumulator) IsEmpty

func (a *LinkAccumulator) IsEmpty() bool

func (*LinkAccumulator) Reset

func (a *LinkAccumulator) Reset()

type LinkBuilder

type LinkBuilder struct {
	// contains filtered or unexported fields
}

LinkBuilder is an Arrow builder for Link records.

func NewLinkBuilder

func NewLinkBuilder(rBuilder *builder.RecordBuilderExt, conf *LinkConfig) *LinkBuilder

func (*LinkBuilder) Accumulator

func (b *LinkBuilder) Accumulator() *LinkAccumulator

func (*LinkBuilder) Build

func (b *LinkBuilder) Build() (record arrow.Record, err error)

func (*LinkBuilder) IsEmpty

func (b *LinkBuilder) IsEmpty() bool

func (*LinkBuilder) PayloadType

func (b *LinkBuilder) PayloadType() *acommon.PayloadType

func (*LinkBuilder) Release

func (b *LinkBuilder) Release()

Release releases the memory allocated by the builder.

func (*LinkBuilder) Reset

func (b *LinkBuilder) Reset()

func (*LinkBuilder) Schema

func (b *LinkBuilder) Schema() *arrow.Schema

func (*LinkBuilder) SchemaID

func (b *LinkBuilder) SchemaID() string

func (*LinkBuilder) SetAttributesAccumulator

func (b *LinkBuilder) SetAttributesAccumulator(accu *acommon.Attributes32Accumulator)

func (*LinkBuilder) TryBuild

func (b *LinkBuilder) TryBuild(attrsAccu *acommon.Attributes32Accumulator) (record arrow.Record, err error)

type LinkConfig

type LinkConfig struct {
	Sorter LinkSorter
}

type LinkSorter

type LinkSorter interface {
	Sort(links []*Link)
	Encode(parentID uint16, link *Link) uint16
	Reset()
}

type LinkStats

type LinkStats struct {
	TotalCount   int64
	Distribution *hdrhistogram.Histogram
	TraceID      *hyperloglog.Sketch
	SpanID       *hyperloglog.Sketch
	TraceState   *hyperloglog.Sketch
	Attributes   *carrow.AttributesStats
}

func NewLinkStats

func NewLinkStats() *LinkStats

func (*LinkStats) ShowStats

func (l *LinkStats) ShowStats(indent string)

func (*LinkStats) UpdateWith

func (l *LinkStats) UpdateWith(links ptrace.SpanLinkSlice, dac uint32)

type LinksByNothing

type LinksByNothing struct{}
func UnsortedLinks() *LinksByNothing

func (*LinksByNothing) Encode

func (s *LinksByNothing) Encode(parentID uint16, _ *Link) uint16

func (*LinksByNothing) Reset

func (s *LinksByNothing) Reset()

func (*LinksByNothing) Sort

func (s *LinksByNothing) Sort(_ []*Link)

type LinksByTraceIdParentId

type LinksByTraceIdParentId struct {
	// contains filtered or unexported fields
}

func SortLinksByTraceIdParentId

func SortLinksByTraceIdParentId() *LinksByTraceIdParentId

func (*LinksByTraceIdParentId) Encode

func (s *LinksByTraceIdParentId) Encode(parentID uint16, link *Link) uint16

func (*LinksByTraceIdParentId) IsSameGroup

func (s *LinksByTraceIdParentId) IsSameGroup(link *Link) bool

func (*LinksByTraceIdParentId) Reset

func (s *LinksByTraceIdParentId) Reset()

func (*LinksByTraceIdParentId) Sort

func (s *LinksByTraceIdParentId) Sort(links []*Link)

type RelatedData

type RelatedData struct {
	// contains filtered or unexported fields
}

RelatedData is a collection of related/dependent data to span entities.

func NewRelatedData

func NewRelatedData(cfg *Config, stats *stats.ProducerStats) (*RelatedData, error)

func (*RelatedData) AttrsBuilders

func (r *RelatedData) AttrsBuilders() *AttrsBuilders

func (*RelatedData) BuildRecordMessages

func (r *RelatedData) BuildRecordMessages() ([]*record_message.RecordMessage, error)

func (*RelatedData) EventBuilder

func (r *RelatedData) EventBuilder() *EventBuilder

func (*RelatedData) LinkBuilder

func (r *RelatedData) LinkBuilder() *LinkBuilder

func (*RelatedData) NextSpanID

func (r *RelatedData) NextSpanID() uint16

func (*RelatedData) RecordBuilderExt

func (r *RelatedData) RecordBuilderExt(payloadType *carrow.PayloadType) *builder.RecordBuilderExt

func (*RelatedData) Release

func (r *RelatedData) Release()

func (*RelatedData) Reset

func (r *RelatedData) Reset()

func (*RelatedData) Schemas

func (r *RelatedData) Schemas() []carrow.SchemaWithPayload

func (*RelatedData) SpanCount

func (r *RelatedData) SpanCount() uint16

type ResourceSpansStats

type ResourceSpansStats struct {
	TotalCount          int64
	Distribution        *hdrhistogram.Histogram
	ResSpansIDsDistinct *hyperloglog.Sketch
	ResourceStats       *carrow.ResourceStats
	ScopeSpansStats     *ScopeSpansStats
	SchemaUrlStats      *carrow.SchemaUrlStats
}

func (*ResourceSpansStats) ShowStats

func (r *ResourceSpansStats) ShowStats(indent string)

func (*ResourceSpansStats) UpdateWith

func (r *ResourceSpansStats) UpdateWith(traces *TracesOptimized)

type ScopeSpansStats

type ScopeSpansStats struct {
	Distribution          *hdrhistogram.Histogram
	ScopeSpansIDsDistinct *hyperloglog.Sketch
	ScopeStats            *carrow.ScopeStats
	SchemaUrlStats        *carrow.SchemaUrlStats
	SpanStats             *SpanStats
}

func (*ScopeSpansStats) ShowStats

func (s *ScopeSpansStats) ShowStats(indent string)

type SpanConfig

type SpanConfig struct {
	Sorter SpanSorter
}

type SpanSorter

type SpanSorter interface {
	Sort(spans []*FlattenedSpan)
}

type SpanStats

type SpanStats struct {
	TotalCount        int64
	Distribution      *hdrhistogram.Histogram
	Attributes        *carrow.AttributesStats
	SharedAttributes  *carrow.AttributesStats
	TimeIntervalStats *carrow.TimeIntervalStats
	Name              *carrow.StringStats
	SpanID            *hyperloglog.Sketch
	TraceID           *hyperloglog.Sketch
	ParentSpanID      *hyperloglog.Sketch
	Kind              *hyperloglog.Sketch
	TraceState        *hyperloglog.Sketch
	Events            *EventStats
	DropEventsCount   *hyperloglog.Sketch
	Links             *LinkStats
	DropLinksCount    *hyperloglog.Sketch
	StatusStats       *carrow.StatusStats
}

func NewSpanStats

func NewSpanStats() *SpanStats

func (*SpanStats) ShowStats

func (s *SpanStats) ShowStats(indent string)

type SpansByNothing

type SpansByNothing struct{}

func UnsortedSpans

func UnsortedSpans() *SpansByNothing

func (*SpansByNothing) Sort

func (s *SpansByNothing) Sort(_ []*FlattenedSpan)

type SpansByResourceSpanIdScopeSpanIdNameStartTimestamp

type SpansByResourceSpanIdScopeSpanIdNameStartTimestamp struct{}

func SortSpansByResourceSpanIdScopeSpanIdNameStartTimestamp

func SortSpansByResourceSpanIdScopeSpanIdNameStartTimestamp() *SpansByResourceSpanIdScopeSpanIdNameStartTimestamp

func (*SpansByResourceSpanIdScopeSpanIdNameStartTimestamp) Sort

type SpansByResourceSpanIdScopeSpanIdNameTraceId

type SpansByResourceSpanIdScopeSpanIdNameTraceId struct{}

func SortSpansByResourceSpanIdScopeSpanIdNameTraceId

func SortSpansByResourceSpanIdScopeSpanIdNameTraceId() *SpansByResourceSpanIdScopeSpanIdNameTraceId

func (*SpansByResourceSpanIdScopeSpanIdNameTraceId) Sort

type SpansByResourceSpanIdScopeSpanIdNameTraceIdStartTimestamp

type SpansByResourceSpanIdScopeSpanIdNameTraceIdStartTimestamp struct{}

func (*SpansByResourceSpanIdScopeSpanIdNameTraceIdStartTimestamp) Sort

type SpansByResourceSpanIdScopeSpanIdStartTimestampNameTraceId

type SpansByResourceSpanIdScopeSpanIdStartTimestampNameTraceId struct{}

func (*SpansByResourceSpanIdScopeSpanIdStartTimestampNameTraceId) Sort

type SpansByResourceSpanIdScopeSpanIdStartTimestampTraceIdName

type SpansByResourceSpanIdScopeSpanIdStartTimestampTraceIdName struct{}

func (*SpansByResourceSpanIdScopeSpanIdStartTimestampTraceIdName) Sort

type SpansByResourceSpanIdScopeSpanIdTraceIdName

type SpansByResourceSpanIdScopeSpanIdTraceIdName struct{}

func SortSpansByResourceSpanIdScopeSpanIdTraceIdName

func SortSpansByResourceSpanIdScopeSpanIdTraceIdName() *SpansByResourceSpanIdScopeSpanIdTraceIdName

func (*SpansByResourceSpanIdScopeSpanIdTraceIdName) Sort

type StatusBuilder

type StatusBuilder struct {
	// contains filtered or unexported fields
}

func StatusBuilderFrom

func StatusBuilderFrom(sb *builder.StructBuilder) *StatusBuilder

func (*StatusBuilder) Append

func (b *StatusBuilder) Append(status ptrace.Status) error

Append appends a new span status to the builder.

func (*StatusBuilder) Build

func (b *StatusBuilder) Build() (*array.Struct, error)

Build builds the span status array struct.

Once the array is no longer needed, Release() must be called to free the memory allocated by the array.

func (*StatusBuilder) Release

func (b *StatusBuilder) Release()

Release releases the memory allocated by the builder.

type TracesAnalyzer

type TracesAnalyzer struct {
	TraceCount         int64
	ResourceSpansStats *ResourceSpansStats
}

func NewTraceAnalyzer

func NewTraceAnalyzer() *TracesAnalyzer

func (*TracesAnalyzer) Analyze

func (t *TracesAnalyzer) Analyze(traces *TracesOptimized)

func (*TracesAnalyzer) ShowStats

func (t *TracesAnalyzer) ShowStats(indent string)

type TracesBuilder

type TracesBuilder struct {
	// contains filtered or unexported fields
}

TracesBuilder is a helper to build a list of resource spans.

func NewTracesBuilder

func NewTracesBuilder(
	rBuilder *builder.RecordBuilderExt,
	cfg *Config,
	stats *stats.ProducerStats,
) (*TracesBuilder, error)

NewTracesBuilder creates a new TracesBuilder with a given allocator.

func (*TracesBuilder) Append

func (b *TracesBuilder) Append(traces ptrace.Traces) error

Append appends a new set of resource spans to the builder.

func (*TracesBuilder) Build

func (b *TracesBuilder) Build() (record arrow.Record, err error)

Build builds an Arrow Record from the builder.

Once the array is no longer needed, Release() must be called to free the memory allocated by the record.

This method returns a DictionaryOverflowError if the cardinality of a dictionary (or several) exceeds the maximum allowed value.

func (*TracesBuilder) RelatedData

func (b *TracesBuilder) RelatedData() *RelatedData

func (*TracesBuilder) Release

func (b *TracesBuilder) Release()

Release releases the memory allocated by the builder.

func (*TracesBuilder) ShowSchema

func (b *TracesBuilder) ShowSchema()

type TracesOptimized

type TracesOptimized struct {
	Spans []*FlattenedSpan
}

type TracesOptimizer

type TracesOptimizer struct {
	// contains filtered or unexported fields
}

func NewTracesOptimizer

func NewTracesOptimizer(sorter SpanSorter) *TracesOptimizer

func (*TracesOptimizer) Optimize

func (t *TracesOptimizer) Optimize(traces ptrace.Traces) *TracesOptimized

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL