cluster

Documentation

Index

• Constants
• Variables
• func ReadTLV(r io.Reader) (byte, []byte, error)
• func WriteTLV(w io.Writer, typ byte, buf []byte) error
• type Config
  • func NewConfig() Config
• type ConsistencyLevel
• type PointsWriter
  • func NewPointsWriter() *PointsWriter
  • func (w *PointsWriter) Close() error
  • func (w *PointsWriter) MapShards(wp *WritePointsRequest) (*ShardMapping, error)
  • func (w *PointsWriter) Open() error
  • func (w *PointsWriter) WritePoints(p *WritePointsRequest) error
• type Service
  • func NewService(c Config) *Service
  • func (s *Service) Close() error
  • func (s *Service) Open() error
  • func (s *Service) SetLogger(l *log.Logger)
• type ShardMapping
  • func NewShardMapping() *ShardMapping
  • func (s *ShardMapping) MapPoint(shardInfo *meta.ShardInfo, p tsdb.Point)
• type ShardWriter
  • func NewShardWriter(timeout time.Duration) *ShardWriter
  • func (w *ShardWriter) Close() error
  • func (w *ShardWriter) WriteShard(shardID, ownerID uint64, points []tsdb.Point) error
• type WritePointsRequest
  • func (w *WritePointsRequest) AddPoint(name string, value interface{}, timestamp time.Time, tags map[string]string)
• type WriteShardRequest
  • func (w *WriteShardRequest) AddPoint(name string, value interface{}, timestamp time.Time, tags map[string]string)
  • func (w *WriteShardRequest) AddPoints(points []tsdb.Point)
  • func (w *WriteShardRequest) MarshalBinary() ([]byte, error)
  • func (w *WriteShardRequest) Points() []tsdb.Point
  • func (w *WriteShardRequest) SetShardID(id uint64)
  • func (w *WriteShardRequest) ShardID() uint64
  • func (w *WriteShardRequest) UnmarshalBinary(buf []byte) error
• type WriteShardResponse
  • func (w *WriteShardResponse) Code() int
  • func (w *WriteShardResponse) MarshalBinary() ([]byte, error)
  • func (w *WriteShardResponse) Message() string
  • func (w *WriteShardResponse) SetCode(code int)
  • func (w *WriteShardResponse) SetMessage(message string)
  • func (w *WriteShardResponse) UnmarshalBinary(buf []byte) error

Constants

const (
	// DefaultShardWriterTimeout is the default timeout set on shard writers.
	DefaultShardWriterTimeout = 5 * time.Second
)

const DefaultWriteTimeout = 5 * time.Second

const MaxMessageSize = 1024 * 1024 * 1024 // 1GB

MaxMessageSize defines how large a message can be before we reject it

const MuxHeader = 2

MuxHeader is the header byte used in the TCP mux.

Variables

var (
	// ErrTimeout is returned when a write times out.
	ErrTimeout = errors.New("timeout")

	// ErrPartialWrite is returned when a write partially succeeds but does
	// not meet the requested consistency level.
	ErrPartialWrite = errors.New("partial write")

	// ErrWriteFailed is returned when no writes succeeded.
	ErrWriteFailed = errors.New("write failed")
)

Functions

func ReadTLV

func ReadTLV(r io.Reader) (byte, []byte, error)

ReadTLV reads a type-length-value record from r.

func WriteTLV

func WriteTLV(w io.Writer, typ byte, buf []byte) error

WriteTLV writes a type-length-value record to w.

Types

type Config

type Config struct {
	ShardWriterTimeout toml.Duration `toml:"shard-writer-timeout"`
}

Config represents the configuration for the the clustering service.

func NewConfig

func NewConfig() Config

NewConfig returns an instance of Config with defaults.

type ConsistencyLevel

type ConsistencyLevel int

ConsistencyLevel represent a required replication criteria before a write can be returned as successful

const (
	// ConsistencyLevelAny allows for hinted hand off, potentially no write happened yet
	ConsistencyLevelAny ConsistencyLevel = iota

	// ConsistencyLevelOne requires at least one data node acknowledged a write
	ConsistencyLevelOne

	// ConsistencyLevelOne requires a quorum of data nodes to acknowledge a write
	ConsistencyLevelQuorum

	// ConsistencyLevelAll requires all data nodes to acknowledge a write
	ConsistencyLevelAll
)

type PointsWriter

type PointsWriter struct {
	MetaStore interface {
		NodeID() uint64
		Database(name string) (di *meta.DatabaseInfo, err error)
		RetentionPolicy(database, policy string) (*meta.RetentionPolicyInfo, error)
		CreateShardGroupIfNotExists(database, policy string, timestamp time.Time) (*meta.ShardGroupInfo, error)
		ShardOwner(shardID uint64) (string, string, *meta.ShardGroupInfo)
	}

	TSDBStore interface {
		CreateShard(database, retentionPolicy string, shardID uint64) error
		WriteToShard(shardID uint64, points []tsdb.Point) error
	}

	ShardWriter interface {
		WriteShard(shardID, ownerID uint64, points []tsdb.Point) error
	}

	HintedHandoff interface {
		WriteShard(shardID, ownerID uint64, points []tsdb.Point) error
	}
	// contains filtered or unexported fields
}

PointsWriter handles writes across multiple local and remote data nodes.

func NewPointsWriter

func NewPointsWriter() *PointsWriter

NewPointsWriter returns a new instance of PointsWriter for a node.

func (*PointsWriter) Close

func (w *PointsWriter) Close() error

func (*PointsWriter) MapShards

func (w *PointsWriter) MapShards(wp *WritePointsRequest) (*ShardMapping, error)

MapShards maps the points contained in wp to a ShardMapping. If a point maps to a shard group or shard that does not currently exist, it will be created before returning the mapping.

func (*PointsWriter) Open

func (w *PointsWriter) Open() error

func (*PointsWriter) WritePoints

func (w *PointsWriter) WritePoints(p *WritePointsRequest) error

WritePoints writes across multiple local and remote data nodes according the consistency level.

type Service

type Service struct {
	Listener net.Listener

	MetaStore interface {
		ShardOwner(shardID uint64) (string, string, *meta.ShardGroupInfo)
	}

	TSDBStore interface {
		CreateShard(database, policy string, shardID uint64) error
		WriteToShard(shardID uint64, points []tsdb.Point) error
	}

	Logger *log.Logger
	// contains filtered or unexported fields
}

Service processes data received over raw TCP connections.

func NewService

func NewService(c Config) *Service

NewService returns a new instance of Service.

func (*Service) Close

func (s *Service) Close() error

Close shuts down the listener and waits for all connections to finish.

func (*Service) Open

func (s *Service) Open() error

Open opens the network listener and begins serving requests.

func (*Service) SetLogger

func (s *Service) SetLogger(l *log.Logger)

SetLogger sets the internal logger to the logger passed in.

type ShardMapping

type ShardMapping struct {
	Points map[uint64][]tsdb.Point    // The points associated with a shard ID
	Shards map[uint64]*meta.ShardInfo // The shards that have been mapped, keyed by shard ID
}

ShardMapping contains a mapping of a shards to a points.

func NewShardMapping

func NewShardMapping() *ShardMapping

NewShardMapping creates an empty ShardMapping

func (*ShardMapping) MapPoint

func (s *ShardMapping) MapPoint(shardInfo *meta.ShardInfo, p tsdb.Point)

MapPoint maps a point to shard

type ShardWriter

type ShardWriter struct {
	MetaStore interface {
		Node(id uint64) (ni *meta.NodeInfo, err error)
	}
	// contains filtered or unexported fields
}

ShardWriter writes a set of points to a shard.

func NewShardWriter

func NewShardWriter(timeout time.Duration) *ShardWriter

NewShardWriter returns a new instance of ShardWriter.

func (*ShardWriter) Close

func (w *ShardWriter) Close() error

func (*ShardWriter) WriteShard

func (w *ShardWriter) WriteShard(shardID, ownerID uint64, points []tsdb.Point) error

type WritePointsRequest

type WritePointsRequest struct {
	Database         string
	RetentionPolicy  string
	ConsistencyLevel ConsistencyLevel
	Points           []tsdb.Point
}

WritePointsRequest represents a request to write point data to the cluster

func (*WritePointsRequest) AddPoint

func (w *WritePointsRequest) AddPoint(name string, value interface{}, timestamp time.Time, tags map[string]string)

AddPoint adds a point to the WritePointRequest with field name 'value'

type WriteShardRequest

type WriteShardRequest struct {
	// contains filtered or unexported fields
}

WriteShardRequest represents the a request to write a slice of points to a shard

func (*WriteShardRequest) AddPoint

func (w *WriteShardRequest) AddPoint(name string, value interface{}, timestamp time.Time, tags map[string]string)

func (*WriteShardRequest) AddPoints

func (w *WriteShardRequest) AddPoints(points []tsdb.Point)

func (*WriteShardRequest) MarshalBinary

func (w *WriteShardRequest) MarshalBinary() ([]byte, error)

MarshalBinary encodes the object to a binary format.

func (*WriteShardRequest) Points

func (w *WriteShardRequest) Points() []tsdb.Point

func (*WriteShardRequest) SetShardID

func (w *WriteShardRequest) SetShardID(id uint64)

func (*WriteShardRequest) ShardID

func (w *WriteShardRequest) ShardID() uint64

func (*WriteShardRequest) UnmarshalBinary

func (w *WriteShardRequest) UnmarshalBinary(buf []byte) error

UnmarshalBinary populates WritePointRequest from a binary format.

type WriteShardResponse

type WriteShardResponse struct {
	// contains filtered or unexported fields
}

WriteShardResponse represents the response returned from a remote WriteShardRequest call

func (*WriteShardResponse) Code

func (w *WriteShardResponse) Code() int

func (*WriteShardResponse) MarshalBinary

func (w *WriteShardResponse) MarshalBinary() ([]byte, error)

MarshalBinary encodes the object to a binary format.

func (*WriteShardResponse) Message

func (w *WriteShardResponse) Message() string

func (*WriteShardResponse) SetCode

func (w *WriteShardResponse) SetCode(code int)

func (*WriteShardResponse) SetMessage

func (w *WriteShardResponse) SetMessage(message string)

func (*WriteShardResponse) UnmarshalBinary

func (w *WriteShardResponse) UnmarshalBinary(buf []byte) error

UnmarshalBinary populates WritePointRequest from a binary format.