config

package
v1.6.2-0...-8a36637 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Nov 27, 2024 License: AGPL-3.0 Imports: 19 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	StorageTypeAlibabaCloud   = "alibabacloud"
	StorageTypeAWS            = "aws"
	StorageTypeAWSDynamo      = "aws-dynamo"
	StorageTypeAzure          = "azure"
	StorageTypeBOS            = "bos"
	StorageTypeBoltDB         = "boltdb"
	StorageTypeCassandra      = "cassandra"
	StorageTypeInMemory       = "inmemory"
	StorageTypeBigTable       = "bigtable"
	StorageTypeBigTableHashed = "bigtable-hashed"
	StorageTypeFileSystem     = "filesystem"
	StorageTypeGCP            = "gcp"
	StorageTypeGCPColumnKey   = "gcp-columnkey"
	StorageTypeGCS            = "gcs"
	StorageTypeGrpc           = "grpc-store"
	StorageTypeLocal          = "local"
	StorageTypeS3             = "s3"
	StorageTypeSwift          = "swift"
	StorageTypeCOS            = "cos"
	// BoltDBShipperType holds the index type for using boltdb with shipper which keeps flushing them to a shared storage
	BoltDBShipperType = "boltdb-shipper"
	TSDBType          = "tsdb"

	// ObjectStorageIndexRequiredPeriod defines the required index period for object storage based index stores like boltdb-shipper and tsdb
	ObjectStorageIndexRequiredPeriod = 24 * time.Hour
)

Variables

This section is empty.

Functions

func ActivePeriodConfig 

func ActivePeriodConfig(configs []PeriodConfig) int

ActivePeriodConfig returns index of active PeriodicConfig which would be applicable to logs that would be pushed starting now. Note: Another PeriodicConfig might be applicable for future logs which can change index type.

func ExtractTableNumberFromName 

func ExtractTableNumberFromName(tableName string) (int64, error)

ExtractTableNumberFromName extracts the table number from a given tableName. returns -1 on error.

func IsObjectStorageIndex 

func IsObjectStorageIndex(indexType string) bool

IsObjectStorageIndex returns true if the index type is either boltdb-shipper or tsdb.

func UsingObjectStorageIndex 

func UsingObjectStorageIndex(configs []PeriodConfig) bool

UsingObjectStorageIndex returns true if the current or any of the upcoming periods use an object store index.

func ValidatePathPrefix 

func ValidatePathPrefix(prefix string) error

Types

type ActiveTableProvisionConfig 

type ActiveTableProvisionConfig struct {
	ProvisionedThroughputOnDemandMode bool  `yaml:"enable_ondemand_throughput_mode"`
	ProvisionedWriteThroughput        int64 `yaml:"provisioned_write_throughput"`
	ProvisionedReadThroughput         int64 `yaml:"provisioned_read_throughput"`

	WriteScale AutoScalingConfig `yaml:"write_scale"`
	ReadScale  AutoScalingConfig `yaml:"read_scale"`
}

func (ActiveTableProvisionConfig) BuildTableDesc 

func (cfg ActiveTableProvisionConfig) BuildTableDesc(tableName string, tags Tags) TableDesc

func (*ActiveTableProvisionConfig) RegisterFlags 

func (cfg *ActiveTableProvisionConfig) RegisterFlags(argPrefix string, f *flag.FlagSet)

RegisterFlags adds the flags required to config this to the given FlagSet. Make sure defaults are set in the respective fields before calling RegisterFlags.

type AutoScalingConfig 

type AutoScalingConfig struct {
	Enabled     bool    `yaml:"enabled"`
	RoleARN     string  `yaml:"role_arn"`
	MinCapacity int64   `yaml:"min_capacity"`
	MaxCapacity int64   `yaml:"max_capacity"`
	OutCooldown int64   `yaml:"out_cooldown"`
	InCooldown  int64   `yaml:"in_cooldown"`
	TargetValue float64 `yaml:"target"`
}

AutoScalingConfig for DynamoDB tables.

func (*AutoScalingConfig) RegisterFlags 

func (cfg *AutoScalingConfig) RegisterFlags(argPrefix string, f *flag.FlagSet)

RegisterFlags adds the flags required to config this to the given FlagSet.

type ChunkStoreConfig 

type ChunkStoreConfig struct {
	ChunkCacheConfig       cache.Config `yaml:"chunk_cache_config"`
	ChunkCacheConfigL2     cache.Config `yaml:"chunk_cache_config_l2" doc:"hidden"`
	WriteDedupeCacheConfig cache.Config `` /* 274-byte string literal not displayed */

	L2ChunkCacheHandoff   time.Duration  `yaml:"l2_chunk_cache_handoff" doc:"hidden"`
	CacheLookupsOlderThan model.Duration `yaml:"cache_lookups_older_than"`

	// When DisableIndexDeduplication is true and chunk is already there in cache, only index would be written to the store and not chunk.
	DisableIndexDeduplication bool `yaml:"-"`
	// contains filtered or unexported fields
}

func (*ChunkStoreConfig) ChunkCacheStubs 

func (cfg *ChunkStoreConfig) ChunkCacheStubs() bool

func (*ChunkStoreConfig) RegisterFlags 

func (cfg *ChunkStoreConfig) RegisterFlags(f *flag.FlagSet)

RegisterFlags adds the flags required to configure this flag set.

func (*ChunkStoreConfig) Validate 

func (cfg *ChunkStoreConfig) Validate() error

type DayTable 

type DayTable struct {
	DayTime
	Prefix string
}

func NewDayTable 

func NewDayTable(d DayTime, prefix string) DayTable

func (DayTable) Addr 

func (d DayTable) Addr() string

Addr returns the prefix (if any) and the unix day offset as a string, which is used as the address for the index table in storage.

func (DayTable) String 

func (d DayTable) String() string

type DayTime 

type DayTime struct {
	model.Time
}

DayTime is a model.Time what holds day-aligned values, and marshals to/from YAML in YYYY-MM-DD format.

func NewDayTime 

func NewDayTime(d model.Time) DayTime

func (DayTime) After 

func (d DayTime) After(other DayTime) bool

func (DayTime) Before 

func (d DayTime) Before(other DayTime) bool

func (DayTime) Bounds 

func (d DayTime) Bounds() (model.Time, model.Time)

func (DayTime) Dec 

func (d DayTime) Dec() DayTime

func (DayTime) Inc 

func (d DayTime) Inc() DayTime

func (DayTime) MarshalYAML 

func (d DayTime) MarshalYAML() (interface{}, error)

MarshalYAML implements yaml.Marshaller.

func (DayTime) ModelTime 

func (d DayTime) ModelTime() model.Time

func (DayTime) String 

func (d DayTime) String() string

func (*DayTime) UnmarshalYAML 

func (d *DayTime) UnmarshalYAML(unmarshal func(interface{}) error) error

UnmarshalYAML implements yaml.Unmarshaller.

type InactiveTableProvisionConfig 

type InactiveTableProvisionConfig struct {
	InactiveThroughputOnDemandMode bool  `yaml:"enable_inactive_throughput_on_demand_mode"`
	InactiveWriteThroughput        int64 `yaml:"inactive_write_throughput"`
	InactiveReadThroughput         int64 `yaml:"inactive_read_throughput"`

	InactiveWriteScale AutoScalingConfig `yaml:"inactive_write_scale"`
	InactiveReadScale  AutoScalingConfig `yaml:"inactive_read_scale"`

	InactiveWriteScaleLastN int64 `yaml:"inactive_write_scale_lastn"`
	InactiveReadScaleLastN  int64 `yaml:"inactive_read_scale_lastn"`
}

func (InactiveTableProvisionConfig) BuildTableDesc 

func (cfg InactiveTableProvisionConfig) BuildTableDesc(tableName string, tags Tags, disableAutoscale bool) TableDesc

func (*InactiveTableProvisionConfig) RegisterFlags 

func (cfg *InactiveTableProvisionConfig) RegisterFlags(argPrefix string, f *flag.FlagSet)

RegisterFlags adds the flags required to config this to the given FlagSet.

type IndexPeriodicTableConfig 

type IndexPeriodicTableConfig struct {
	PathPrefix          string `` /* 168-byte string literal not displayed */
	PeriodicTableConfig `yaml:",inline"`
}

func (IndexPeriodicTableConfig) MarshalYAML 

func (cfg IndexPeriodicTableConfig) MarshalYAML() (interface{}, error)

MarshalYAML implements the yaml.Marshaler interface.

func (*IndexPeriodicTableConfig) UnmarshalYAML 

func (cfg *IndexPeriodicTableConfig) UnmarshalYAML(unmarshal func(interface{}) error) error

UnmarshalYAML implements the yaml.Unmarshaler interface.

func (*IndexPeriodicTableConfig) Validate 

func (cfg *IndexPeriodicTableConfig) Validate() error

type PeriodConfig 

type PeriodConfig struct {
	// used when working with config
	From DayTime `` /* 261-byte string literal not displayed */
	// type of index client to use.
	IndexType string `` /* 267-byte string literal not displayed */
	// type of object client to use.
	ObjectType  string                   `` /* 308-byte string literal not displayed */
	Schema      string                   `yaml:"schema" doc:"description=The schema version to use, current recommended schema is v12."`
	IndexTables IndexPeriodicTableConfig `yaml:"index" doc:"description=Configures how the index is updated and stored."`
	ChunkTables PeriodicTableConfig      `yaml:"chunks" doc:"description=Configured how the chunks are updated and stored."`
	RowShards   uint32                   `yaml:"row_shards" doc:"default=16|description=How many shards will be created. Only used if schema is v10 or greater."`
	// contains filtered or unexported fields
}

PeriodConfig defines the schema and tables to use for a period of time

func (*PeriodConfig) ChunkFormat 

func (cfg *PeriodConfig) ChunkFormat() (byte, chunkenc.HeadBlockFmt, error)

ChunkFormat returns chunk format including it's headBlockFormat corresponding to the `schema` version in the given `PeriodConfig`.

func (*PeriodConfig) GetIndexTableNumberRange 

func (cfg *PeriodConfig) GetIndexTableNumberRange(schemaEndDate DayTime) TableRange

GetIndexTableNumberRange returns the table number range calculated based on the configured schema start date, index table period and the given schemaEndDate

func (*PeriodConfig) TSDBFormat 

func (cfg *PeriodConfig) TSDBFormat() (int, error)

TSDBFormat returns index format corresponding to the `schema` version in the given `PeriodConfig`.

func (*PeriodConfig) UnmarshalYAML 

func (cfg *PeriodConfig) UnmarshalYAML(unmarshal func(interface{}) error) error

UnmarshalYAML implements yaml.Unmarshaller.

func (*PeriodConfig) VersionAsInt 

func (cfg *PeriodConfig) VersionAsInt() (int, error)

type PeriodicTableConfig 

type PeriodicTableConfig struct {
	Prefix string        `yaml:"prefix" doc:"description=Table prefix for all period tables."`
	Period time.Duration `yaml:"period" doc:"description=Table period."`
	Tags   Tags          `yaml:"tags" doc:"description=A map to be added to all managed tables."`
}

PeriodicTableConfig is configuration for a set of time-sharded tables.

func (PeriodicTableConfig) MarshalYAML 

func (cfg PeriodicTableConfig) MarshalYAML() (interface{}, error)

MarshalYAML implements the yaml.Marshaler interface.

func (*PeriodicTableConfig) PeriodicTables 

func (cfg *PeriodicTableConfig) PeriodicTables(from, through model.Time, pCfg ProvisionConfig, beginGrace, endGrace time.Duration, retention time.Duration) []TableDesc

func (*PeriodicTableConfig) TableFor 

func (cfg *PeriodicTableConfig) TableFor(t model.Time) string

TableFor calculates the table shard for a given point in time.

func (*PeriodicTableConfig) UnmarshalYAML 

func (cfg *PeriodicTableConfig) UnmarshalYAML(unmarshal func(interface{}) error) error

UnmarshalYAML implements the yaml.Unmarshaler interface.

func (PeriodicTableConfig) Validate 

func (cfg PeriodicTableConfig) Validate() error

type ProvisionConfig 

type ProvisionConfig struct {
	ActiveTableProvisionConfig   `yaml:",inline"`
	InactiveTableProvisionConfig `yaml:",inline"`
}

ProvisionConfig holds config for provisioning capacity for index and chunk tables (on DynamoDB for now)

func (*ProvisionConfig) RegisterFlags 

func (cfg *ProvisionConfig) RegisterFlags(argPrefix string, f *flag.FlagSet)

RegisterFlags adds the flags required to config this to the given FlagSet.

type SchemaConfig 

type SchemaConfig struct {
	Configs []PeriodConfig `yaml:"configs"`
	// contains filtered or unexported fields
}

SchemaConfig contains the config for our chunk index schemas

func (SchemaConfig) ChunkTableFor 

func (cfg SchemaConfig) ChunkTableFor(t model.Time) (string, error)

ChunkTableFor calculates the chunk table shard for a given point in time.

func (SchemaConfig) ExternalKey 

func (cfg SchemaConfig) ExternalKey(ref logproto.ChunkRef) string

Generate the appropriate external key based on cfg.Schema, chunk.Checksum, and chunk.From

func (*SchemaConfig) ForEachAfter 

func (cfg *SchemaConfig) ForEachAfter(t model.Time, f func(config *PeriodConfig))

ForEachAfter will call f() on every entry after t, splitting entries if necessary so there is an entry starting at t

func (*SchemaConfig) Load 

func (cfg *SchemaConfig) Load() error

Load the yaml file, or build the config from legacy command-line flags

func (*SchemaConfig) RegisterFlags 

func (cfg *SchemaConfig) RegisterFlags(f *flag.FlagSet)

RegisterFlags adds the flags required to config this to the given FlagSet.

func (SchemaConfig) SchemaForTime 

func (cfg SchemaConfig) SchemaForTime(t model.Time) (PeriodConfig, error)

SchemaForTime returns the Schema PeriodConfig to use for a given point in time.

func (*SchemaConfig) Validate 

func (cfg *SchemaConfig) Validate() error

Validate the schema config and returns an error if the validation doesn't pass

func (SchemaConfig) VersionForChunk 

func (cfg SchemaConfig) VersionForChunk(ref logproto.ChunkRef) int

VersionForChunk will return the schema version associated with the `From` timestamp of a chunk. The schema and chunk must be valid+compatible as the errors are not checked.

type TableDesc 

type TableDesc struct {
	Name              string
	UseOnDemandIOMode bool
	ProvisionedRead   int64
	ProvisionedWrite  int64
	Tags              Tags
	WriteScale        AutoScalingConfig
	ReadScale         AutoScalingConfig
}

TableDesc describes a table.

func (TableDesc) Equals 

func (desc TableDesc) Equals(other TableDesc) bool

Equals returns true if other matches desc.

type TableRange 

type TableRange struct {
	Start, End   int64
	PeriodConfig *PeriodConfig
}

TableRange represents a range of table numbers built based on the configured schema start/end date and the table period. Both Start and End are inclusive.

func (TableRange) ConfigForTableNumber 

func (t TableRange) ConfigForTableNumber(tableNumber int64) *PeriodConfig

func (TableRange) TableInRange 

func (t TableRange) TableInRange(tableName string) (bool, error)

TableInRange tells whether given table falls in the range and the tableName has the right prefix based on the schema config.

type TableRanges 

type TableRanges []TableRange

TableRanges represents a list of table ranges for multiple schemas.

func GetIndexStoreTableRanges 

func GetIndexStoreTableRanges(indexType string, periodicConfigs []PeriodConfig) TableRanges

func (TableRanges) ConfigForTableNumber 

func (t TableRanges) ConfigForTableNumber(tableNumber int64) *PeriodConfig

func (TableRanges) TableInRange 

func (t TableRanges) TableInRange(tableName string) (bool, error)

TableInRange tells whether given table falls in any of the ranges and the tableName has the right prefix based on the schema config.

func (TableRanges) TableNameFor 

func (t TableRanges) TableNameFor(table int64) (string, bool)

type Tags 

type Tags map[string]string

Tags is a string-string map that implements flag.Value.

func (Tags) Equals 

func (ts Tags) Equals(other Tags) bool

Equals returns true is other matches ts.

func (*Tags) Set 

func (ts *Tags) Set(s string) error

Set implements flag.Value

func (Tags) String 

func (ts Tags) String() string

String implements flag.Value

func (*Tags) UnmarshalYAML 

func (ts *Tags) UnmarshalYAML(unmarshal func(interface{}) error) error

UnmarshalYAML implements yaml.Unmarshaler.

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.