config

package
v0.4.1 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Aug 27, 2018 License: Apache-2.0 Imports: 29 Imported by: 11

Documentation

Index

Constants

View Source 
const (
	// DefaultNewFileMode is the default new file mode.
	DefaultNewFileMode = os.FileMode(0666)
	// DefaultNewDirectoryMode is the default new directory mode.
	DefaultNewDirectoryMode = os.FileMode(0755)
)

Variables

This section is empty.

Functions

func InitialClusterEndpoints 

func InitialClusterEndpoints(initialCluster []environment.SeedNode) ([]string, error)

InitialClusterEndpoints returns the endpoints of the initial cluster

func IsSeedNode 

func IsSeedNode(initialCluster []environment.SeedNode, hostID string) bool

IsSeedNode returns whether the given hostID is an etcd node.

func NewEtcdEmbedConfig 

func NewEtcdEmbedConfig(cfg DBConfiguration) (*embed.Config, error)

NewEtcdEmbedConfig creates a new embedded etcd config from kv config.

func ValidateBootstrappersOrder 

func ValidateBootstrappersOrder(names []string) error

ValidateBootstrappersOrder will validate that a list of bootstrappers specified is in valid order.

Types

type BlockRetrievePolicy 

type BlockRetrievePolicy struct {
	// FetchConcurrency is the concurrency to fetch blocks from disk. For
	// spinning disks it is highly recommended to set this value to 1.
	FetchConcurrency int `yaml:"fetchConcurrency" validate:"min=0"`
}

BlockRetrievePolicy is the block retrieve policy.

type BootstrapConfiguration 

type BootstrapConfiguration struct {
	// Bootstrappers is the list of bootstrappers, ordered by precedence in
	// descending order.
	Bootstrappers []string `yaml:"bootstrappers" validate:"nonzero"`

	// Filesystem bootstrapper configuration.
	Filesystem *BootstrapFilesystemConfiguration `yaml:"fs"`

	// Peers bootstrapper configuration.
	Peers *BootstrapPeersConfiguration `yaml:"peers"`

	// CacheSeriesMetadata determines whether individual bootstrappers cache
	// series metadata across all calls (namespaces / shards / blocks).
	CacheSeriesMetadata *bool `yaml:"cacheSeriesMetadata"`
}

BootstrapConfiguration specifies the config for bootstrappers.

func (BootstrapConfiguration) New 

func (bsc BootstrapConfiguration) New(
	opts storage.Options,
	adminClient client.AdminClient,
) (bootstrap.ProcessProvider, error)

New creates a bootstrap process based on the bootstrap configuration.

type BootstrapFilesystemConfiguration 

type BootstrapFilesystemConfiguration struct {
	// NumProcessorsPerCPU is the number of processors per CPU.
	NumProcessorsPerCPU float64 `yaml:"numProcessorsPerCPU" validate:"min=0.0"`
}

BootstrapFilesystemConfiguration specifies config for the fs bootstrapper.

type BootstrapPeersConfiguration 

type BootstrapPeersConfiguration struct {
	// FetchBlocksMetadataEndpointVersion is the endpoint to use when fetching blocks metadata.
	// TODO: Remove once v1 endpoint no longer required.
	FetchBlocksMetadataEndpointVersion client.FetchBlocksMetadataEndpointVersion `yaml:"fetchBlocksMetadataEndpointVersion"`
}

BootstrapPeersConfiguration specifies config for the peers bootstrapper.

type BucketPoolPolicy 

type BucketPoolPolicy struct {
	// The pool buckets sizes to use
	Buckets []CapacityPoolPolicy `yaml:"buckets"`
}

BucketPoolPolicy specifies a bucket pool policy.

type CacheConfigurations 

type CacheConfigurations struct {
	// Series cache policy.
	Series *SeriesCacheConfiguration `yaml:"series"`
}

CacheConfigurations is the cache configurations.

func (CacheConfigurations) SeriesConfiguration 

func (c CacheConfigurations) SeriesConfiguration() SeriesCacheConfiguration

SeriesConfiguration returns the series cache configuration or default if none is specified.

type CalculationType 

type CalculationType string

CalculationType is a type of configuration parameter. 

const (
	// CalculationTypeFixed is a fixed parameter not to be scaled of any parameter.
	CalculationTypeFixed CalculationType = "fixed"
	// CalculationTypePerCPU is a parameter that needs to be scaled by number of CPUs.
	CalculationTypePerCPU CalculationType = "percpu"
)

type CapacityPoolPolicy 

type CapacityPoolPolicy struct {
	// The size of the pool
	Size int `yaml:"size"`

	// The capacity of items in the pool
	Capacity int `yaml:"capacity"`

	// The low watermark to start refilling the pool, if zero none
	RefillLowWaterMark float64 `yaml:"lowWatermark" validate:"min=0.0,max=1.0"`

	// The high watermark to stop refilling the pool, if zero none
	RefillHighWaterMark float64 `yaml:"highWatermark" validate:"min=0.0,max=1.0"`
}

CapacityPoolPolicy specifies a single pool policy that has a per element capacity.

type CommitLogPolicy 

type CommitLogPolicy struct {
	// The max size the commit log will flush a segment to disk after buffering.
	FlushMaxBytes int `yaml:"flushMaxBytes" validate:"nonzero"`

	// The maximum amount of time the commit log will wait to flush to disk.
	FlushEvery time.Duration `yaml:"flushEvery" validate:"nonzero"`

	// The queue the commit log will keep in front of the current commit log segment.
	Queue CommitLogQueuePolicy `yaml:"queue" validate:"nonzero"`

	// The commit log retention policy.
	RetentionPeriod time.Duration `yaml:"retentionPeriod" validate:"nonzero"`

	// The commit log block size.
	BlockSize time.Duration `yaml:"blockSize" validate:"nonzero"`
}

CommitLogPolicy is the commit log policy.

type CommitLogQueuePolicy 

type CommitLogQueuePolicy struct {
	// The type of calculation for the size.
	CalculationType CalculationType `yaml:"calculationType"`

	// The size of the commit log, calculated according to the calculation type.
	Size int `yaml:"size" validate:"nonzero"`
}

CommitLogQueuePolicy is the commit log queue policy.

type Configuration 

type Configuration struct {
	// DB is the configuration for a DB node (required).
	DB *DBConfiguration `yaml:"db"`

	// Coordinator is the configuration for the coordinator to run (optional).
	Coordinator *coordinatorcfg.Configuration `yaml:"coordinator"`
}

Configuration is the top level configuration that includes both a DB node and a coordinator.

type ContextPoolPolicy 

type ContextPoolPolicy struct {
	// The size of the pool
	Size int `yaml:"size"`

	// The low watermark to start refilling the pool, if zero none
	RefillLowWaterMark float64 `yaml:"lowWatermark" validate:"min=0.0,max=1.0"`

	// The high watermark to stop refilling the pool, if zero none
	RefillHighWaterMark float64 `yaml:"highWatermark" validate:"min=0.0,max=1.0"`

	// The maximum allowable size for a slice of finalizers that the
	// pool will allow to be returned (finalizer slices that grow too
	// large during use will be discarded instead of returning to the
	// pool where they would consume more memory.)
	MaxFinalizerCapacity int `yaml:"maxFinalizerCapacity" validate:"min=0"`
}

ContextPoolPolicy specifies the policy for the context pool

func (ContextPoolPolicy) MaxFinalizerCapacityWithDefault 

func (c ContextPoolPolicy) MaxFinalizerCapacityWithDefault() int

MaxFinalizerCapacityWithDefault returns the maximum finalizer capacity and fallsback to the default value if its not set

func (ContextPoolPolicy) PoolPolicy 

func (c ContextPoolPolicy) PoolPolicy() PoolPolicy

PoolPolicy returns the PoolPolicy that is represented by the ContextPoolPolicy

type DBConfiguration 

type DBConfiguration struct {
	// Index configuration.
	Index IndexConfiguration `yaml:"index"`

	// Logging configuration.
	Logging xlog.Configuration `yaml:"logging"`

	// Metrics configuration.
	Metrics instrument.MetricsConfiguration `yaml:"metrics"`

	// The host and port on which to listen for the node service.
	ListenAddress string `yaml:"listenAddress" validate:"nonzero"`

	// The host and port on which to listen for the cluster service.
	ClusterListenAddress string `yaml:"clusterListenAddress" validate:"nonzero"`

	// The HTTP host and port on which to listen for the node service.
	HTTPNodeListenAddress string `yaml:"httpNodeListenAddress" validate:"nonzero"`

	// The HTTP host and port on which to listen for the cluster service.
	HTTPClusterListenAddress string `yaml:"httpClusterListenAddress" validate:"nonzero"`

	// The host and port on which to listen for debug endpoints.
	DebugListenAddress string `yaml:"debugListenAddress"`

	// HostID is the local host ID configuration.
	HostID hostid.Configuration `yaml:"hostID"`

	// Client configuration, used for inter-node communication and when used as a coordinator.
	Client client.Configuration `yaml:"client"`

	// The initial garbage collection target percentage.
	GCPercentage int `yaml:"gcPercentage" validate:"max=100"`

	// Write new series limit per second to limit overwhelming during new ID bursts.
	WriteNewSeriesLimitPerSecond int `yaml:"writeNewSeriesLimitPerSecond"`

	// Write new series backoff between batches of new series insertions.
	WriteNewSeriesBackoffDuration time.Duration `yaml:"writeNewSeriesBackoffDuration"`

	// The tick configuration, omit this to use default settings.
	Tick *TickConfiguration `yaml:"tick"`

	// Bootstrap configuration.
	Bootstrap BootstrapConfiguration `yaml:"bootstrap"`

	// The block retriever policy.
	BlockRetrieve *BlockRetrievePolicy `yaml:"blockRetrieve"`

	// Cache configurations.
	Cache CacheConfigurations `yaml:"cache"`

	// The filesystem configuration for the node.
	Filesystem FilesystemConfiguration `yaml:"fs"`

	// The commit log policy for the node.
	CommitLog CommitLogPolicy `yaml:"commitlog"`

	// The repair policy for repairing in-memory data.
	Repair RepairPolicy `yaml:"repair"`

	// The pooling policy.
	PoolingPolicy PoolingPolicy `yaml:"pooling"`

	// The environment (static or dynamic) configuration.
	EnvironmentConfig environment.Configuration `yaml:"config"`

	// The configuration for hashing
	Hashing HashingConfiguration `yaml:"hashing"`

	// Write new series asynchronously for fast ingestion of new ID bursts.
	WriteNewSeriesAsync bool `yaml:"writeNewSeriesAsync"`
}

DBConfiguration is the configuration for a DB node.

type FilesystemConfiguration 

type FilesystemConfiguration struct {
	// File path prefix for reading/writing TSDB files
	FilePathPrefix string `yaml:"filePathPrefix" validate:"nonzero"`

	// Write buffer size
	WriteBufferSize int `yaml:"writeBufferSize" validate:"min=1"`

	// Data read buffer size
	DataReadBufferSize int `yaml:"dataReadBufferSize" validate:"min=1"`

	// Info metadata file read buffer size
	InfoReadBufferSize int `yaml:"infoReadBufferSize" validate:"min=1"`

	// Seek data read buffer size
	SeekReadBufferSize int `yaml:"seekReadBufferSize" validate:"min=1"`

	// Disk flush throughput limit in Mb/s
	ThroughputLimitMbps float64 `yaml:"throughputLimitMbps" validate:"min=0.0"`

	// Disk flush throughput check interval
	ThroughputCheckEvery int `yaml:"throughputCheckEvery" validate:"nonzero"`

	// NewFileMode is the new file permissions mode to use when
	// creating files - specify as three digits, e.g. 666.
	NewFileMode *string `yaml:"newFileMode"`

	// NewDirectoryMode is the new file permissions mode to use when
	// creating directories - specify as three digits, e.g. 755.
	NewDirectoryMode *string `yaml:"newDirectoryMode"`

	// Mmap is the mmap options which features are primarily platform dependent
	Mmap *MmapConfiguration `yaml:"mmap"`
}

FilesystemConfiguration is the filesystem configuration.

func (FilesystemConfiguration) MmapConfiguration 

func (p FilesystemConfiguration) MmapConfiguration() MmapConfiguration

MmapConfiguration returns the effective mmap configuration.

func (FilesystemConfiguration) ParseNewDirectoryMode 

func (p FilesystemConfiguration) ParseNewDirectoryMode() (os.FileMode, error)

ParseNewDirectoryMode parses the specified new directory mode.

func (FilesystemConfiguration) ParseNewFileMode 

func (p FilesystemConfiguration) ParseNewFileMode() (os.FileMode, error)

ParseNewFileMode parses the specified new file mode.

type HashingConfiguration 

type HashingConfiguration struct {
	// Murmur32 seed value.
	Seed uint32 `yaml:"seed"`
}

HashingConfiguration is the configuration for hashing.

type IndexConfiguration added in v0.3.0 

type IndexConfiguration struct {
	// MaxQueryIDsConcurrency controls the maximum number of outstanding QueryID
	// requests that can be serviced concurrently. Limiting the concurrency is
	// important to prevent index queries from overloading the database entirely
	// as they are very CPU-intensive (regex and FST matching.)
	MaxQueryIDsConcurrency int `yaml:"maxQueryIDsConcurrency" validate:"min=0"`
}

IndexConfiguration contains index-specific configuration.

type LRUSeriesCachePolicyConfiguration 

type LRUSeriesCachePolicyConfiguration struct {
	MaxBlocks uint `yaml:"maxBlocks" validate:"nonzero"`
}

LRUSeriesCachePolicyConfiguration contains configuration for the LRU series caching policy.

type MaxCapacityPoolPolicy 

type MaxCapacityPoolPolicy struct {
	// The size of the pool
	Size int `yaml:"size"`

	// The capacity of items in the pool
	Capacity int `yaml:"capacity"`

	// The max capacity of items in the pool
	MaxCapacity int `yaml:"maxCapacity"`

	// The low watermark to start refilling the pool, if zero none
	RefillLowWaterMark float64 `yaml:"lowWatermark" validate:"min=0.0,max=1.0"`

	// The high watermark to stop refilling the pool, if zero none
	RefillHighWaterMark float64 `yaml:"highWatermark" validate:"min=0.0,max=1.0"`
}

MaxCapacityPoolPolicy specifies a single pool policy that has a per element capacity, and a maximum allowed capacity as well.

type MmapConfiguration 

type MmapConfiguration struct {
	// HugeTLB is the huge pages configuration which will only take affect
	// on platforms that support it, currently just linux
	HugeTLB MmapHugeTLBConfiguration `yaml:"hugeTLB"`
}

MmapConfiguration is the mmap configuration.

func DefaultMmapConfiguration 

func DefaultMmapConfiguration() MmapConfiguration

DefaultMmapConfiguration is the default mmap configuration.

type MmapHugeTLBConfiguration 

type MmapHugeTLBConfiguration struct {
	// Enabled if true or disabled if false
	Enabled bool `yaml:"enabled"`

	// Threshold is the threshold on which to use the huge TLB flag if enabled
	Threshold int64 `yaml:"threshold"`
}

MmapHugeTLBConfiguration is the mmap huge TLB configuration.

type PoolPolicy 

type PoolPolicy struct {
	// The size of the pool
	Size int `yaml:"size"`

	// The low watermark to start refilling the pool, if zero none
	RefillLowWaterMark float64 `yaml:"lowWatermark" validate:"min=0.0,max=1.0"`

	// The high watermark to stop refilling the pool, if zero none
	RefillHighWaterMark float64 `yaml:"highWatermark" validate:"min=0.0,max=1.0"`
}

PoolPolicy specifies a single pool policy.

type PoolingPolicy 

type PoolingPolicy struct {
	// The initial alloc size for a block
	BlockAllocSize int `yaml:"blockAllocSize"`

	// The general pool type: simple or native.
	Type PoolingType `yaml:"type"`

	// The Bytes pool buckets to use
	BytesPool BucketPoolPolicy `yaml:"bytesPool"`

	// The policy for the Closers pool
	ClosersPool PoolPolicy `yaml:"closersPool"`

	// The policy for the Context pool
	ContextPool ContextPoolPolicy `yaml:"contextPool"`

	// The policy for the DatabaseSeries pool
	SeriesPool PoolPolicy `yaml:"seriesPool"`

	// The policy for the DatabaseBlock pool
	BlockPool PoolPolicy `yaml:"blockPool"`

	// The policy for the Encoder pool
	EncoderPool PoolPolicy `yaml:"encoderPool"`

	// The policy for the Iterator pool
	IteratorPool PoolPolicy `yaml:"iteratorPool"`

	// The policy for the Segment Reader pool
	SegmentReaderPool PoolPolicy `yaml:"segmentReaderPool"`

	// The policy for the Identifier pool
	IdentifierPool PoolPolicy `yaml:"identifierPool"`

	// The policy for the FetchBlockMetadataResult pool
	FetchBlockMetadataResultsPool CapacityPoolPolicy `yaml:"fetchBlockMetadataResultsPool"`

	// The policy for the FetchBlocksMetadataResults pool
	FetchBlocksMetadataResultsPool CapacityPoolPolicy `yaml:"fetchBlocksMetadataResultsPool"`

	// The policy for the HostBlockMetadataSlice pool
	HostBlockMetadataSlicePool CapacityPoolPolicy `yaml:"hostBlockMetadataSlicePool"`

	// The policy for the BlockMetadat pool
	BlockMetadataPool PoolPolicy `yaml:"blockMetadataPool"`

	// The policy for the BlockMetadataSlice pool
	BlockMetadataSlicePool CapacityPoolPolicy `yaml:"blockMetadataSlicePool"`

	// The policy for the BlocksMetadata pool
	BlocksMetadataPool PoolPolicy `yaml:"blocksMetadataPool"`

	// The policy for the BlocksMetadataSlice pool
	BlocksMetadataSlicePool CapacityPoolPolicy `yaml:"blocksMetadataSlicePool"`

	// The policy for the tags pool
	TagsPool MaxCapacityPoolPolicy `yaml:"tagsPool"`

	// The policy for the tags iterator pool
	TagsIteratorPool PoolPolicy `yaml:"tagIteratorPool"`

	// The policy for the index.ResultsPool
	IndexResultsPool PoolPolicy `yaml:"indexResultsPool"`

	// The policy for the TagEncoderPool
	TagEncoderPool PoolPolicy `yaml:"tagEncoderPool"`

	// The policy for the TagDecoderPool
	TagDecoderPool PoolPolicy `yaml:"tagDecoderPool"`
}

PoolingPolicy specifies the pooling policy.

type PoolingType 

type PoolingType string

PoolingType is a type of pooling, using runtime or mmap'd bytes pooling. 

const (
	// SimplePooling uses the basic Go runtime to allocate bytes for bytes pools.
	SimplePooling PoolingType = "simple"
	// NativePooling uses a mmap syscall to allocate bytes for bytes pools, take
	// great care when experimenting with this. There's not enough protection
	// even with ref counting that M3DB performs to use this safely in
	// production. Here be dragons and so forth.
	NativePooling PoolingType = "native"
)

type RepairPolicy 

type RepairPolicy struct {
	// Enabled or disabled.
	Enabled bool `yaml:"enabled"`

	// The repair interval.
	Interval time.Duration `yaml:"interval" validate:"nonzero"`

	// The repair time offset.
	Offset time.Duration `yaml:"offset" validate:"nonzero"`

	// The repair time jitter.
	Jitter time.Duration `yaml:"jitter" validate:"nonzero"`

	// The repair throttle.
	Throttle time.Duration `yaml:"throttle" validate:"nonzero"`

	// The repair check interval.
	CheckInterval time.Duration `yaml:"checkInterval" validate:"nonzero"`
}

RepairPolicy is the repair policy.

type SeriesCacheConfiguration 

type SeriesCacheConfiguration struct {
	Policy series.CachePolicy                 `yaml:"policy"`
	LRU    *LRUSeriesCachePolicyConfiguration `yaml:"lru"`
}

SeriesCacheConfiguration is the series cache configuration.

type TickConfiguration 

type TickConfiguration struct {
	// Tick series batch size is the batch size to process series together
	// during a tick before yielding and sleeping the per series duration
	// multiplied by the batch size.
	// The higher this value is the more variable CPU utilization will be
	// but the shorter ticks will ultimately be.
	SeriesBatchSize int `yaml:"seriesBatchSize"`

	// Tick per series sleep at the completion of a tick batch.
	PerSeriesSleepDuration time.Duration `yaml:"perSeriesSleepDuration"`

	// Tick minimum interval controls the minimum tick interval for the node.
	MinimumInterval time.Duration `yaml:"minimumInterval"`
}

TickConfiguration is the tick configuration for background processing of series as blocks are rotated from mutable to immutable and out of order writes are merged.

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.