protocol

Documentation

Index

• Constants
• type ConnectionHeader
• type ControlMessage
• type DataMessage
• type FieldMapping
• type IndexDescriptor
  • func (d IndexDescriptor) Validate() error
• type MappingDescriptor
  • func (md *MappingDescriptor) UnmarshalJSON(data []byte) error
  • func (d MappingDescriptor) Validate() error

Constants

const (
	ControlMessageACK       = "ack"
	ControlMessageBACKOFF   = "bo"
	ControlMessageRECONNECT = "re"
	ControlMessageERROR     = "error"
	ControlMessageREADY     = "rd"
	ControlMessageFLOW      = "fl"
)

const (
	CurrentVersion = 2
)

Types

type ConnectionHeader

type ConnectionHeader struct {
	Version         int                 `json:"version"`
	Oid             string              `json:"oid"`
	InstallationKey string              `json:"iid"`
	Hostname        string              `json:"host_name,omitempty"`
	Platform        string              `json:"platform"`
	Architecture    string              `json:"architecture"`
	Mapping         MappingDescriptor   `json:"mapping,omitempty"`
	Mappings        []MappingDescriptor `json:"mappings,omitempty"`
	SensorSeedKey   string              `json:"sensor_seed_key"`
	IsCompressed    bool                `json:"is_compressed"`
	DataFormat      string              `json:"data_format"` // LimaCharlie supports "msgpack" or "json"
	InstanceID      string              `json:"instance_id"`
	Indexing        []IndexDescriptor   `json:"indexing,omitempty"`
}

type ControlMessage

type ControlMessage struct {
	Verb string `json:"v"`

	// Optional components depending on verb.
	SeqNum     uint64 `json:"x,omitempty"`
	Duration   uint64 `json:"dur,omitempty"`
	WindowSize uint64 `json:"win,omitempty"`
	Error      string `json:"err,omitempty"`
}

type DataMessage

type DataMessage struct {
	SeqNum       uint64 `json:"x" msgpack:"x"`                     // Populated by the USP Client.
	AckRequested bool   `json:"a,omitempty" msgpack:"a,omitempty"` // Populated by the USP Client.

	// One of, payload format.
	TextPayload             string                 `json:"t,omitempty" msgpack:"t,omitempty"`
	JsonPayload             map[string]interface{} `json:"j,omitempty" msgpack:"j,omitempty"`
	BinaryPayload           string                 `json:"b,omitempty" msgpack:"b,omitempty"`
	BundlePayload           []byte                 `json:"-" msgpack:"o,omitempty"` // Bundle of records
	CompressedBundlePayload []byte                 `json:"-" msgpack:"p,omitempty"` // Compressed bundle or records

	// Basic event metadata.
	EventType   string `json:"e,omitempty" msgpack:"e,omitempty"`
	TimestampMs uint64 `json:"s,omitempty" msgpack:"s,omitempty"`

	// Optional contextual information.
	ThisAtom        string `json:"this,omitempty" msgpack:"this,omitempty"`
	ParentAtom      string `json:"parent,omitempty" msgpack:"parent,omitempty"`
	TargetAtom      string `json:"target,omitempty" msgpack:"target,omitempty"`
	InvestigationID string `json:"inv,omitempty" msgpack:"inv,omitempty"`
}

type FieldMapping

type FieldMapping struct {
	// Map the source field to the destination field.
	SourceField      string `json:"src_field" yaml:"src_field"`
	DestinationField string `json:"dst_field" yaml:"dst_field"`
}

Deprecated: field mappings are now deprecated in favor of transforms.

type IndexDescriptor

type IndexDescriptor struct {
	// If either is specified, this descriptor only applies to
	// the specified event_types, or to all event_types excluding
	// some specific ones.
	EventsIncluded []string `json:"events_included,omitempty" yaml:"events_included,omitempty"`
	EventsExcluded []string `json:"events_excluded,omitempty" yaml:"events_excluded,omitempty"`

	// Path within the relevant events to use as indexed field.
	// Like: metadata/user/user_name
	Path string `json:"path" yaml:"path"`

	// Regexp is a regular expression that applies to the element
	// in Path to extract the indexed value. It should be a regexp
	// in the `re2` format: https://github.com/google/re2/wiki/Syntax
	// with a single capture group (the value to be indexed).
	Regexp string `json:"regexp,omitempty" yaml:"regexp,omitempty"`

	// What index the above field belongs to. Unsupported indexes
	// will result in an error.
	// Like: user
	IndexType string `json:"index_type" yaml:"index_type"`
}

Indexing descriptors are applied in the LimaCharlie cloud once all mapping and parsing operations have been applied.

func (IndexDescriptor) Validate

func (d IndexDescriptor) Validate() error

type MappingDescriptor

type MappingDescriptor struct {
	// Use the named capture groups from the regular
	// expression below to parse text lines into JSON.
	ParsingRE string `json:"parsing_re,omitempty" yaml:"parsing_re,omitempty"`

	// Path to the component of the JSON events that
	// indicates unique values to become Sensor IDs.
	SensorKeyPath string `json:"sensor_key_path,omitempty" yaml:"sensor_key_path,omitempty"`

	// Path to the component of the JSON events that
	// indicates the hostname of the sensor per the SensorKeyPath.
	SensorHostnamePath string `json:"sensor_hostname_path,omitempty" yaml:"sensor_hostname_path,omitempty"`

	// Path to the component that should be used as
	// the Event Type of a specific event.
	EventTypePath string `json:"event_type_path,omitempty" yaml:"event_type_path,omitempty"`

	// Path to the component that should be used as
	// the Event Time (converted to a ms epoch).
	EventTimePath string `json:"event_time_path,omitempty" yaml:"event_time_path,omitempty"`

	// Path to the component that should be used as
	// the Investigation ID of a specific event.
	InvestigationIDPath string `json:"investigation_id_path,omitempty" yaml:"investigation_id_path,omitempty"`

	// Rename the fields in the Mappings but leave
	// all other fields in place. The default behavior
	// is to replace the final event with only the
	// mapped values.
	// Deprecated: field mappings are now deprecated in favor of transforms.
	IsRenameOnly bool `json:"rename_only,omitempty" yaml:"rename_only,omitempty"`
	// Deprecated: field mappings are now deprecated in favor of transforms.
	Mappings []FieldMapping `json:"mappings,omitempty" yaml:"mappings,omitempty"`

	// Transform applied to the events.
	Transform map[string]interface{} `json:"transform,omitempty" yaml:"transform,omitempty"`

	// List of field paths to drop upon ingestion.
	DropFields []string `json:"drop_fields,omitempty" yaml:"drop_fields,omitempty"`
}

func (*MappingDescriptor) UnmarshalJSON

func (md *MappingDescriptor) UnmarshalJSON(data []byte) error

func (MappingDescriptor) Validate

func (d MappingDescriptor) Validate() error