config

package
v0.5.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Feb 3, 2019 License: Apache-2.0 Imports: 31 Imported by: 11

Documentation

Index

Constants

View Source 
const (
	// DefaultNewFileMode is the default new file mode.
	DefaultNewFileMode = os.FileMode(0666)
	// DefaultNewDirectoryMode is the default new directory mode.
	DefaultNewDirectoryMode = os.FileMode(0755)
)

Variables

This section is empty.

Functions

func InitialClusterEndpoints 

func InitialClusterEndpoints(initialCluster []environment.SeedNode) ([]string, error)

InitialClusterEndpoints returns the endpoints of the initial cluster

func IsSeedNode 

func IsSeedNode(initialCluster []environment.SeedNode, hostID string) bool

IsSeedNode returns whether the given hostID is an etcd node.

func NewEtcdEmbedConfig 

func NewEtcdEmbedConfig(cfg DBConfiguration) (*embed.Config, error)

NewEtcdEmbedConfig creates a new embedded etcd config from kv config.

func ValidateBootstrappersOrder 

func ValidateBootstrappersOrder(names []string) error

ValidateBootstrappersOrder will validate that a list of bootstrappers specified is in valid order.

Types

type BlockRetrievePolicy 

type BlockRetrievePolicy struct {
	// FetchConcurrency is the concurrency to fetch blocks from disk. For
	// spinning disks it is highly recommended to set this value to 1.
	FetchConcurrency int `yaml:"fetchConcurrency" validate:"min=0"`
}

BlockRetrievePolicy is the block retrieve policy.

type BootstrapCommitlogConfiguration added in v0.4.8 

type BootstrapCommitlogConfiguration struct {
	// ReturnUnfulfilledForCorruptCommitlogFiles controls whether the commitlog bootstrapper
	// will return unfulfilled for all shard time ranges when it encounters a corrupt commit
	// file. Note that regardless of this value, the commitlog bootstrapper will still try and
	// read all the uncorrupted commitlog files and return as much data as it can, but setting
	// this to true allows the node to attempt a repair if the peers bootstrapper is configured
	// after the commitlog bootstrapper.
	ReturnUnfulfilledForCorruptCommitlogFiles bool `yaml:"returnUnfulfilledForCorruptCommitlogFiles"`
}

BootstrapCommitlogConfiguration specifies config for the commitlog bootstrapper.

type BootstrapConfiguration 

type BootstrapConfiguration struct {
	// Bootstrappers is the list of bootstrappers, ordered by precedence in
	// descending order.
	Bootstrappers []string `yaml:"bootstrappers" validate:"nonzero"`

	// Filesystem bootstrapper configuration.
	Filesystem *BootstrapFilesystemConfiguration `yaml:"fs"`

	// Commitlog bootstrapper configuration.
	Commitlog *BootstrapCommitlogConfiguration `yaml:"commitlog"`

	// CacheSeriesMetadata determines whether individual bootstrappers cache
	// series metadata across all calls (namespaces / shards / blocks).
	CacheSeriesMetadata *bool `yaml:"cacheSeriesMetadata"`
}

BootstrapConfiguration specifies the config for bootstrappers.

func (BootstrapConfiguration) New 

func (bsc BootstrapConfiguration) New(
	opts storage.Options,
	topoMapProvider topology.MapProvider,
	origin topology.Host,
	adminClient client.AdminClient,
) (bootstrap.ProcessProvider, error)

New creates a bootstrap process based on the bootstrap configuration.

type BootstrapFilesystemConfiguration 

type BootstrapFilesystemConfiguration struct {
	// NumProcessorsPerCPU is the number of processors per CPU.
	NumProcessorsPerCPU float64 `yaml:"numProcessorsPerCPU" validate:"min=0.0"`
}

BootstrapFilesystemConfiguration specifies config for the fs bootstrapper.

type BucketPoolPolicy 

type BucketPoolPolicy struct {
	// The pool buckets sizes to use
	Buckets []CapacityPoolPolicy `yaml:"buckets"`
}

BucketPoolPolicy specifies a bucket pool policy.

type CacheConfigurations 

type CacheConfigurations struct {
	// Series cache policy.
	Series *SeriesCacheConfiguration `yaml:"series"`
}

CacheConfigurations is the cache configurations.

func (CacheConfigurations) SeriesConfiguration 

func (c CacheConfigurations) SeriesConfiguration() SeriesCacheConfiguration

SeriesConfiguration returns the series cache configuration or default if none is specified.

type CalculationType 

type CalculationType string

CalculationType is a type of configuration parameter. 

const (
	// CalculationTypeFixed is a fixed parameter not to be scaled of any parameter.
	CalculationTypeFixed CalculationType = "fixed"
	// CalculationTypePerCPU is a parameter that needs to be scaled by number of CPUs.
	CalculationTypePerCPU CalculationType = "percpu"
)

type CapacityPoolPolicy 

type CapacityPoolPolicy struct {
	// The size of the pool.
	Size int `yaml:"size"`

	// The capacity of items in the pool.
	Capacity int `yaml:"capacity"`

	// The low watermark to start refilling the pool, if zero none.
	RefillLowWaterMark float64 `yaml:"lowWatermark" validate:"min=0.0,max=1.0"`

	// The high watermark to stop refilling the pool, if zero none.
	RefillHighWaterMark float64 `yaml:"highWatermark" validate:"min=0.0,max=1.0"`
}

CapacityPoolPolicy specifies a single pool policy that has a per element capacity.

type CommitLogPolicy 

type CommitLogPolicy struct {
	// The max size the commit log will flush a segment to disk after buffering.
	FlushMaxBytes int `yaml:"flushMaxBytes" validate:"nonzero"`

	// The maximum amount of time the commit log will wait to flush to disk.
	FlushEvery time.Duration `yaml:"flushEvery" validate:"nonzero"`

	// The queue the commit log will keep in front of the current commit log segment.
	// Modifying values in this policy will control how many pending writes can be
	// in the commitlog queue before M3DB will begin rejecting writes.
	Queue CommitLogQueuePolicy `yaml:"queue" validate:"nonzero"`

	// The actual Golang channel that implements the commit log queue. We separate this
	// from the Queue field for historical / legacy reasons. Generally speaking, the
	// values in this config should not need to be modified, but we leave it in for
	// tuning purposes. Unlike the Queue field, values in this policy control the size
	// of the channel that backs the queue. Since writes to the commitlog are batched,
	// setting the size of this policy will control how many batches can be queued, and
	// indrectly how many writes can be queued, but that is dependent on the batch size
	// of the client. As a result, we recommend that users avoid tuning this field and
	// modify the Queue size instead which maps directly to the number of writes. This
	// works in most cases because the default size of the QueueChannel should be large
	// enough for almost all workloads assuming a reasonable batch size is used.
	QueueChannel *CommitLogQueuePolicy `yaml:"queueChannel"`

	// The commit log block size.
	BlockSize time.Duration `yaml:"blockSize" validate:"nonzero"`
}

CommitLogPolicy is the commit log policy.

type CommitLogQueuePolicy 

type CommitLogQueuePolicy struct {
	// The type of calculation for the size.
	CalculationType CalculationType `yaml:"calculationType"`

	// The size of the commit log, calculated according to the calculation type.
	Size int `yaml:"size" validate:"nonzero"`
}

CommitLogQueuePolicy is the commit log queue policy.

type Configuration 

type Configuration struct {
	// DB is the configuration for a DB node (required).
	DB *DBConfiguration `yaml:"db"`

	// Coordinator is the configuration for the coordinator to run (optional).
	Coordinator *coordinatorcfg.Configuration `yaml:"coordinator"`
}

Configuration is the top level configuration that includes both a DB node and a coordinator.

type ContextPoolPolicy 

type ContextPoolPolicy struct {
	// The size of the pool
	Size int `yaml:"size"`

	// The low watermark to start refilling the pool, if zero none.
	RefillLowWaterMark float64 `yaml:"lowWatermark" validate:"min=0.0,max=1.0"`

	// The high watermark to stop refilling the pool, if zero none.
	RefillHighWaterMark float64 `yaml:"highWatermark" validate:"min=0.0,max=1.0"`

	// The maximum allowable size for a slice of finalizers that the
	// pool will allow to be returned (finalizer slices that grow too
	// large during use will be discarded instead of returning to the
	// pool where they would consume more memory.)
	MaxFinalizerCapacity int `yaml:"maxFinalizerCapacity" validate:"min=0"`
}

ContextPoolPolicy specifies the policy for the context pool.

func (ContextPoolPolicy) MaxFinalizerCapacityWithDefault 

func (c ContextPoolPolicy) MaxFinalizerCapacityWithDefault() int

MaxFinalizerCapacityWithDefault returns the maximum finalizer capacity and fallsback to the default value if its not set.

func (ContextPoolPolicy) PoolPolicy 

func (c ContextPoolPolicy) PoolPolicy() PoolPolicy

PoolPolicy returns the PoolPolicy that is represented by the ContextPoolPolicy.

type DBConfiguration 

type DBConfiguration struct {
	// Index configuration.
	Index IndexConfiguration `yaml:"index"`

	// Logging configuration.
	Logging xlog.Configuration `yaml:"logging"`

	// Metrics configuration.
	Metrics instrument.MetricsConfiguration `yaml:"metrics"`

	// The host and port on which to listen for the node service.
	ListenAddress string `yaml:"listenAddress" validate:"nonzero"`

	// The host and port on which to listen for the cluster service.
	ClusterListenAddress string `yaml:"clusterListenAddress" validate:"nonzero"`

	// The HTTP host and port on which to listen for the node service.
	HTTPNodeListenAddress string `yaml:"httpNodeListenAddress" validate:"nonzero"`

	// The HTTP host and port on which to listen for the cluster service.
	HTTPClusterListenAddress string `yaml:"httpClusterListenAddress" validate:"nonzero"`

	// The host and port on which to listen for debug endpoints.
	DebugListenAddress string `yaml:"debugListenAddress"`

	// HostID is the local host ID configuration.
	HostID hostid.Configuration `yaml:"hostID"`

	// Client configuration, used for inter-node communication and when used as a coordinator.
	Client client.Configuration `yaml:"client"`

	// The initial garbage collection target percentage.
	GCPercentage int `yaml:"gcPercentage" validate:"max=100"`

	// Write new series limit per second to limit overwhelming during new ID bursts.
	WriteNewSeriesLimitPerSecond int `yaml:"writeNewSeriesLimitPerSecond"`

	// Write new series backoff between batches of new series insertions.
	WriteNewSeriesBackoffDuration time.Duration `yaml:"writeNewSeriesBackoffDuration"`

	// The tick configuration, omit this to use default settings.
	Tick *TickConfiguration `yaml:"tick"`

	// Bootstrap configuration.
	Bootstrap BootstrapConfiguration `yaml:"bootstrap"`

	// The block retriever policy.
	BlockRetrieve *BlockRetrievePolicy `yaml:"blockRetrieve"`

	// Cache configurations.
	Cache CacheConfigurations `yaml:"cache"`

	// The filesystem configuration for the node.
	Filesystem FilesystemConfiguration `yaml:"fs"`

	// The commit log policy for the node.
	CommitLog CommitLogPolicy `yaml:"commitlog"`

	// The repair policy for repairing in-memory data.
	Repair RepairPolicy `yaml:"repair"`

	// The pooling policy.
	PoolingPolicy PoolingPolicy `yaml:"pooling"`

	// The environment (static or dynamic) configuration.
	EnvironmentConfig environment.Configuration `yaml:"config"`

	// The configuration for hashing
	Hashing HashingConfiguration `yaml:"hashing"`

	// Write new series asynchronously for fast ingestion of new ID bursts.
	WriteNewSeriesAsync bool `yaml:"writeNewSeriesAsync"`
}

DBConfiguration is the configuration for a DB node.

type FilesystemConfiguration 

type FilesystemConfiguration struct {
	// File path prefix for reading/writing TSDB files
	FilePathPrefix string `yaml:"filePathPrefix" validate:"nonzero"`

	// Write buffer size
	WriteBufferSize int `yaml:"writeBufferSize" validate:"min=1"`

	// Data read buffer size
	DataReadBufferSize int `yaml:"dataReadBufferSize" validate:"min=1"`

	// Info metadata file read buffer size
	InfoReadBufferSize int `yaml:"infoReadBufferSize" validate:"min=1"`

	// Seek data read buffer size
	SeekReadBufferSize int `yaml:"seekReadBufferSize" validate:"min=1"`

	// Disk flush throughput limit in Mb/s
	ThroughputLimitMbps float64 `yaml:"throughputLimitMbps" validate:"min=0.0"`

	// Disk flush throughput check interval
	ThroughputCheckEvery int `yaml:"throughputCheckEvery" validate:"nonzero"`

	// NewFileMode is the new file permissions mode to use when
	// creating files - specify as three digits, e.g. 666.
	NewFileMode *string `yaml:"newFileMode"`

	// NewDirectoryMode is the new file permissions mode to use when
	// creating directories - specify as three digits, e.g. 755.
	NewDirectoryMode *string `yaml:"newDirectoryMode"`

	// Mmap is the mmap options which features are primarily platform dependent
	Mmap *MmapConfiguration `yaml:"mmap"`

	// ForceIndexSummariesMmapMemory forces the mmap that stores the index lookup bytes
	// to be an anonymous region in memory as opposed to a file-based mmap.
	ForceIndexSummariesMmapMemory bool `yaml:"force_index_summaries_mmap_memory"`

	// ForceBloomFilterMmapMemory forces the mmap that stores the index lookup bytes
	// to be an anonymous region in memory as opposed to a file-based mmap.
	ForceBloomFilterMmapMemory bool `yaml:"force_bloom_filter_mmap_memory"`
}

FilesystemConfiguration is the filesystem configuration.

func (FilesystemConfiguration) MmapConfiguration 

func (p FilesystemConfiguration) MmapConfiguration() MmapConfiguration

MmapConfiguration returns the effective mmap configuration.

func (FilesystemConfiguration) ParseNewDirectoryMode 

func (p FilesystemConfiguration) ParseNewDirectoryMode() (os.FileMode, error)

ParseNewDirectoryMode parses the specified new directory mode.

func (FilesystemConfiguration) ParseNewFileMode 

func (p FilesystemConfiguration) ParseNewFileMode() (os.FileMode, error)

ParseNewFileMode parses the specified new file mode.

type HashingConfiguration 

type HashingConfiguration struct {
	// Murmur32 seed value.
	Seed uint32 `yaml:"seed"`
}

HashingConfiguration is the configuration for hashing.

type IndexConfiguration added in v0.3.0 

type IndexConfiguration struct {
	// MaxQueryIDsConcurrency controls the maximum number of outstanding QueryID
	// requests that can be serviced concurrently. Limiting the concurrency is
	// important to prevent index queries from overloading the database entirely
	// as they are very CPU-intensive (regex and FST matching.)
	MaxQueryIDsConcurrency int `yaml:"maxQueryIDsConcurrency" validate:"min=0"`
}

IndexConfiguration contains index-specific configuration.

type LRUSeriesCachePolicyConfiguration 

type LRUSeriesCachePolicyConfiguration struct {
	MaxBlocks         uint `yaml:"maxBlocks" validate:"nonzero"`
	EventsChannelSize uint `yaml:"eventsChannelSize" validate:"nonzero"`
}

LRUSeriesCachePolicyConfiguration contains configuration for the LRU series caching policy.

type MaxCapacityPoolPolicy 

type MaxCapacityPoolPolicy struct {
	// The size of the pool.
	Size int `yaml:"size"`

	// The capacity of items in the pool.
	Capacity int `yaml:"capacity"`

	// The max capacity of items in the pool.
	MaxCapacity int `yaml:"maxCapacity"`

	// The low watermark to start refilling the pool, if zero none.
	RefillLowWaterMark float64 `yaml:"lowWatermark" validate:"min=0.0,max=1.0"`

	// The high watermark to stop refilling the pool, if zero none.
	RefillHighWaterMark float64 `yaml:"highWatermark" validate:"min=0.0,max=1.0"`
}

MaxCapacityPoolPolicy specifies a single pool policy that has a per element capacity, and a maximum allowed capacity as well.

type MmapConfiguration 

type MmapConfiguration struct {
	// HugeTLB is the huge pages configuration which will only take affect
	// on platforms that support it, currently just linux
	HugeTLB MmapHugeTLBConfiguration `yaml:"hugeTLB"`
}

MmapConfiguration is the mmap configuration.

func DefaultMmapConfiguration 

func DefaultMmapConfiguration() MmapConfiguration

DefaultMmapConfiguration is the default mmap configuration.

type MmapHugeTLBConfiguration 

type MmapHugeTLBConfiguration struct {
	// Enabled if true or disabled if false
	Enabled bool `yaml:"enabled"`

	// Threshold is the threshold on which to use the huge TLB flag if enabled
	Threshold int64 `yaml:"threshold"`
}

MmapHugeTLBConfiguration is the mmap huge TLB configuration.

type PoolPolicy 

type PoolPolicy struct {
	// The size of the pool.
	Size int `yaml:"size"`

	// The low watermark to start refilling the pool, if zero none.
	RefillLowWaterMark float64 `yaml:"lowWatermark" validate:"min=0.0,max=1.0"`

	// The high watermark to stop refilling the pool, if zero none.
	RefillHighWaterMark float64 `yaml:"highWatermark" validate:"min=0.0,max=1.0"`
}

PoolPolicy specifies a single pool policy.

type PoolingPolicy 

type PoolingPolicy struct {
	// The initial alloc size for a block.
	BlockAllocSize int `yaml:"blockAllocSize"`

	// The general pool type (currently only supported: simple).
	Type PoolingType `yaml:"type"`

	// The Bytes pool buckets to use.
	BytesPool BucketPoolPolicy `yaml:"bytesPool"`

	// The policy for the Closers pool.
	ClosersPool PoolPolicy `yaml:"closersPool"`

	// The policy for the Context pool.
	ContextPool ContextPoolPolicy `yaml:"contextPool"`

	// The policy for the DatabaseSeries pool.
	SeriesPool PoolPolicy `yaml:"seriesPool"`

	// The policy for the DatabaseBlock pool.
	BlockPool PoolPolicy `yaml:"blockPool"`

	// The policy for the Encoder pool.
	EncoderPool PoolPolicy `yaml:"encoderPool"`

	// The policy for the Iterator pool.
	IteratorPool PoolPolicy `yaml:"iteratorPool"`

	// The policy for the Segment Reader pool.
	SegmentReaderPool PoolPolicy `yaml:"segmentReaderPool"`

	// The policy for the Identifier pool.
	IdentifierPool PoolPolicy `yaml:"identifierPool"`

	// The policy for the FetchBlockMetadataResult pool.
	FetchBlockMetadataResultsPool CapacityPoolPolicy `yaml:"fetchBlockMetadataResultsPool"`

	// The policy for the FetchBlocksMetadataResults pool.
	FetchBlocksMetadataResultsPool CapacityPoolPolicy `yaml:"fetchBlocksMetadataResultsPool"`

	// The policy for the HostBlockMetadataSlice pool.
	HostBlockMetadataSlicePool CapacityPoolPolicy `yaml:"hostBlockMetadataSlicePool"`

	// The policy for the BlockMetadat pool.
	BlockMetadataPool PoolPolicy `yaml:"blockMetadataPool"`

	// The policy for the BlockMetadataSlice pool.
	BlockMetadataSlicePool CapacityPoolPolicy `yaml:"blockMetadataSlicePool"`

	// The policy for the BlocksMetadata pool.
	BlocksMetadataPool PoolPolicy `yaml:"blocksMetadataPool"`

	// The policy for the BlocksMetadataSlice pool.
	BlocksMetadataSlicePool CapacityPoolPolicy `yaml:"blocksMetadataSlicePool"`

	// The policy for the tags pool.
	TagsPool MaxCapacityPoolPolicy `yaml:"tagsPool"`

	// The policy for the tags iterator pool.
	TagsIteratorPool PoolPolicy `yaml:"tagIteratorPool"`

	// The policy for the index.ResultsPool.
	IndexResultsPool PoolPolicy `yaml:"indexResultsPool"`

	// The policy for the TagEncoderPool.
	TagEncoderPool PoolPolicy `yaml:"tagEncoderPool"`

	// The policy for the TagDecoderPool.
	TagDecoderPool PoolPolicy `yaml:"tagDecoderPool"`

	// The policy for the WriteBatchPool.
	WriteBatchPool WriteBatchPoolPolicy `yaml:"writeBatchPool"`

	// The policy for the PostingsListPool.
	PostingsListPool PoolPolicy `yaml:"postingsListPool"`
}

PoolingPolicy specifies the pooling policy.

func (PoolingPolicy) PostingsListPoolPolicyWithDefaults added in v0.5.0 

func (c PoolingPolicy) PostingsListPoolPolicyWithDefaults() PoolPolicy

PostingsListPoolPolicyWithDefaults returns the postings list pool policy and will set a sensible default size if not specified in the YAML configuration.

type PoolingType 

type PoolingType string

PoolingType is a type of pooling, using runtime or mmap'd bytes pooling. 

const (
	// SimplePooling uses the basic Go runtime to allocate bytes for bytes pools.
	SimplePooling PoolingType = "simple"
)

type RepairPolicy 

type RepairPolicy struct {
	// Enabled or disabled.
	Enabled bool `yaml:"enabled"`

	// The repair interval.
	Interval time.Duration `yaml:"interval" validate:"nonzero"`

	// The repair time offset.
	Offset time.Duration `yaml:"offset" validate:"nonzero"`

	// The repair time jitter.
	Jitter time.Duration `yaml:"jitter" validate:"nonzero"`

	// The repair throttle.
	Throttle time.Duration `yaml:"throttle" validate:"nonzero"`

	// The repair check interval.
	CheckInterval time.Duration `yaml:"checkInterval" validate:"nonzero"`
}

RepairPolicy is the repair policy.

type SeriesCacheConfiguration 

type SeriesCacheConfiguration struct {
	Policy series.CachePolicy                 `yaml:"policy"`
	LRU    *LRUSeriesCachePolicyConfiguration `yaml:"lru"`
}

SeriesCacheConfiguration is the series cache configuration.

type TickConfiguration 

type TickConfiguration struct {
	// Tick series batch size is the batch size to process series together
	// during a tick before yielding and sleeping the per series duration
	// multiplied by the batch size.
	// The higher this value is the more variable CPU utilization will be
	// but the shorter ticks will ultimately be.
	SeriesBatchSize int `yaml:"seriesBatchSize"`

	// Tick per series sleep at the completion of a tick batch.
	PerSeriesSleepDuration time.Duration `yaml:"perSeriesSleepDuration"`

	// Tick minimum interval controls the minimum tick interval for the node.
	MinimumInterval time.Duration `yaml:"minimumInterval"`
}

TickConfiguration is the tick configuration for background processing of series as blocks are rotated from mutable to immutable and out of order writes are merged.

type WriteBatchPoolPolicy added in v0.4.8 

type WriteBatchPoolPolicy struct {
	// InitialBatchSize controls the initial batch size for each WriteBatch when
	// the pool is being constructed / refilled.
	InitialBatchSize *int `yaml:"initialBatchSize"`

	// MaxBatchSize controls the maximum size that a pooled WriteBatch can grow to
	// and still remain in the pool.
	MaxBatchSize *int `yaml:"maxBatchSize"`

	// Pool is the Pooling Policy for the WriteBatch pool.
	Pool PoolPolicy `yaml:"pool"`
}

WriteBatchPoolPolicy specifies the pooling policy for the WriteBatch pool.

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.