Documentation ¶
Index ¶
- Constants
- func ActivePeriodConfig(configs []PeriodConfig) int
- func ExtractTableNumberFromName(tableName string) (int64, error)
- func IsObjectStorageIndex(indexType string) bool
- func UsingObjectStorageIndex(configs []PeriodConfig) bool
- func ValidatePathPrefix(prefix string) error
- type ActiveTableProvisionConfig
- type AutoScalingConfig
- type ChunkStoreConfig
- type DayTable
- type DayTime
- func (d DayTime) After(other DayTime) bool
- func (d DayTime) Before(other DayTime) bool
- func (d DayTime) Bounds() (model.Time, model.Time)
- func (d DayTime) Dec() DayTime
- func (d DayTime) Inc() DayTime
- func (d DayTime) MarshalYAML() (interface{}, error)
- func (d DayTime) ModelTime() model.Time
- func (d DayTime) String() string
- func (d *DayTime) UnmarshalYAML(unmarshal func(interface{}) error) error
- type InactiveTableProvisionConfig
- type IndexPeriodicTableConfig
- type PeriodConfig
- func (cfg *PeriodConfig) ChunkFormat() (byte, chunkenc.HeadBlockFmt, error)
- func (cfg *PeriodConfig) GetIndexTableNumberRange(schemaEndDate DayTime) TableRange
- func (cfg *PeriodConfig) TSDBFormat() (int, error)
- func (cfg *PeriodConfig) UnmarshalYAML(unmarshal func(interface{}) error) error
- func (cfg *PeriodConfig) VersionAsInt() (int, error)
- type PeriodicTableConfig
- func (cfg PeriodicTableConfig) MarshalYAML() (interface{}, error)
- func (cfg *PeriodicTableConfig) PeriodicTables(from, through model.Time, pCfg ProvisionConfig, ...) []TableDesc
- func (cfg *PeriodicTableConfig) TableFor(t model.Time) string
- func (cfg *PeriodicTableConfig) UnmarshalYAML(unmarshal func(interface{}) error) error
- func (cfg PeriodicTableConfig) Validate() error
- type ProvisionConfig
- type SchemaConfig
- func (cfg SchemaConfig) ChunkTableFor(t model.Time) (string, error)
- func (cfg SchemaConfig) ExternalKey(ref logproto.ChunkRef) string
- func (cfg *SchemaConfig) ForEachAfter(t model.Time, f func(config *PeriodConfig))
- func (cfg *SchemaConfig) Load() error
- func (cfg *SchemaConfig) RegisterFlags(f *flag.FlagSet)
- func (cfg SchemaConfig) SchemaForTime(t model.Time) (PeriodConfig, error)
- func (cfg *SchemaConfig) Validate() error
- func (cfg SchemaConfig) VersionForChunk(ref logproto.ChunkRef) int
- type TableDesc
- type TableRange
- type TableRanges
- type Tags
Constants ¶
const ( // ObjectStorageIndexRequiredPeriod defines the required index period for object storage based index stores like boltdb-shipper and tsdb ObjectStorageIndexRequiredPeriod = 24 * time.Hour )
Variables ¶
This section is empty.
Functions ¶
func ActivePeriodConfig ¶
func ActivePeriodConfig(configs []PeriodConfig) int
ActivePeriodConfig returns index of active PeriodicConfig which would be applicable to logs that would be pushed starting now. Note: Another PeriodicConfig might be applicable for future logs which can change index type.
func ExtractTableNumberFromName ¶
ExtractTableNumberFromName extracts the table number from a given tableName. returns -1 on error.
func IsObjectStorageIndex ¶
IsObjectStorageIndex returns true if the index type is either boltdb-shipper or tsdb.
func UsingObjectStorageIndex ¶
func UsingObjectStorageIndex(configs []PeriodConfig) bool
UsingObjectStorageIndex returns true if the current or any of the upcoming periods use an object store index.
func ValidatePathPrefix ¶
Types ¶
type ActiveTableProvisionConfig ¶
type ActiveTableProvisionConfig struct { ProvisionedThroughputOnDemandMode bool `yaml:"enable_ondemand_throughput_mode"` ProvisionedWriteThroughput int64 `yaml:"provisioned_write_throughput"` ProvisionedReadThroughput int64 `yaml:"provisioned_read_throughput"` WriteScale AutoScalingConfig `yaml:"write_scale"` ReadScale AutoScalingConfig `yaml:"read_scale"` }
func (ActiveTableProvisionConfig) BuildTableDesc ¶
func (cfg ActiveTableProvisionConfig) BuildTableDesc(tableName string, tags Tags) TableDesc
func (*ActiveTableProvisionConfig) RegisterFlags ¶
func (cfg *ActiveTableProvisionConfig) RegisterFlags(argPrefix string, f *flag.FlagSet)
RegisterFlags adds the flags required to config this to the given FlagSet. Make sure defaults are set in the respective fields before calling RegisterFlags.
type AutoScalingConfig ¶
type AutoScalingConfig struct { Enabled bool `yaml:"enabled"` RoleARN string `yaml:"role_arn"` MinCapacity int64 `yaml:"min_capacity"` MaxCapacity int64 `yaml:"max_capacity"` OutCooldown int64 `yaml:"out_cooldown"` InCooldown int64 `yaml:"in_cooldown"` TargetValue float64 `yaml:"target"` }
AutoScalingConfig for DynamoDB tables.
func (*AutoScalingConfig) RegisterFlags ¶
func (cfg *AutoScalingConfig) RegisterFlags(argPrefix string, f *flag.FlagSet)
RegisterFlags adds the flags required to config this to the given FlagSet.
type ChunkStoreConfig ¶
type ChunkStoreConfig struct { ChunkCacheConfig cache.Config `yaml:"chunk_cache_config"` ChunkCacheConfigL2 cache.Config `yaml:"chunk_cache_config_l2"` WriteDedupeCacheConfig cache.Config `` /* 274-byte string literal not displayed */ L2ChunkCacheHandoff time.Duration `yaml:"l2_chunk_cache_handoff"` CacheLookupsOlderThan model.Duration `yaml:"cache_lookups_older_than"` // When DisableIndexDeduplication is true and chunk is already there in cache, only index would be written to the store and not chunk. DisableIndexDeduplication bool `yaml:"-"` // contains filtered or unexported fields }
func (*ChunkStoreConfig) ChunkCacheStubs ¶
func (cfg *ChunkStoreConfig) ChunkCacheStubs() bool
func (*ChunkStoreConfig) RegisterFlags ¶
func (cfg *ChunkStoreConfig) RegisterFlags(f *flag.FlagSet)
RegisterFlags adds the flags required to configure this flag set.
func (*ChunkStoreConfig) Validate ¶
func (cfg *ChunkStoreConfig) Validate() error
type DayTable ¶
func NewDayTable ¶
type DayTime ¶
DayTime is a model.Time what holds day-aligned values, and marshals to/from YAML in YYYY-MM-DD format.
func NewDayTime ¶
func (DayTime) MarshalYAML ¶
MarshalYAML implements yaml.Marshaller.
func (*DayTime) UnmarshalYAML ¶
UnmarshalYAML implements yaml.Unmarshaller.
type InactiveTableProvisionConfig ¶
type InactiveTableProvisionConfig struct { InactiveThroughputOnDemandMode bool `yaml:"enable_inactive_throughput_on_demand_mode"` InactiveWriteThroughput int64 `yaml:"inactive_write_throughput"` InactiveReadThroughput int64 `yaml:"inactive_read_throughput"` InactiveWriteScale AutoScalingConfig `yaml:"inactive_write_scale"` InactiveReadScale AutoScalingConfig `yaml:"inactive_read_scale"` InactiveWriteScaleLastN int64 `yaml:"inactive_write_scale_lastn"` InactiveReadScaleLastN int64 `yaml:"inactive_read_scale_lastn"` }
func (InactiveTableProvisionConfig) BuildTableDesc ¶
func (cfg InactiveTableProvisionConfig) BuildTableDesc(tableName string, tags Tags, disableAutoscale bool) TableDesc
func (*InactiveTableProvisionConfig) RegisterFlags ¶
func (cfg *InactiveTableProvisionConfig) RegisterFlags(argPrefix string, f *flag.FlagSet)
RegisterFlags adds the flags required to config this to the given FlagSet.
type IndexPeriodicTableConfig ¶
type IndexPeriodicTableConfig struct { PathPrefix string `` /* 168-byte string literal not displayed */ PeriodicTableConfig `yaml:",inline"` }
func (IndexPeriodicTableConfig) MarshalYAML ¶
func (cfg IndexPeriodicTableConfig) MarshalYAML() (interface{}, error)
MarshalYAML implements the yaml.Marshaler interface.
func (*IndexPeriodicTableConfig) UnmarshalYAML ¶
func (cfg *IndexPeriodicTableConfig) UnmarshalYAML(unmarshal func(interface{}) error) error
UnmarshalYAML implements the yaml.Unmarshaler interface.
func (*IndexPeriodicTableConfig) Validate ¶
func (cfg *IndexPeriodicTableConfig) Validate() error
type PeriodConfig ¶
type PeriodConfig struct { // used when working with config From DayTime `` /* 261-byte string literal not displayed */ // type of index client to use. IndexType string `` /* 267-byte string literal not displayed */ // type of object client to use. ObjectType string `` /* 308-byte string literal not displayed */ Schema string `yaml:"schema" doc:"description=The schema version to use, current recommended schema is v13."` IndexTables IndexPeriodicTableConfig `yaml:"index" doc:"description=Configures how the index is updated and stored."` ChunkTables PeriodicTableConfig `yaml:"chunks" doc:"description=Configured how the chunks are updated and stored."` RowShards uint32 `yaml:"row_shards" doc:"default=16|description=How many shards will be created. Only used if schema is v10 or greater."` // contains filtered or unexported fields }
PeriodConfig defines the schema and tables to use for a period of time
func (*PeriodConfig) ChunkFormat ¶
func (cfg *PeriodConfig) ChunkFormat() (byte, chunkenc.HeadBlockFmt, error)
ChunkFormat returns chunk format including it's headBlockFormat corresponding to the `schema` version in the given `PeriodConfig`.
func (*PeriodConfig) GetIndexTableNumberRange ¶
func (cfg *PeriodConfig) GetIndexTableNumberRange(schemaEndDate DayTime) TableRange
GetIndexTableNumberRange returns the table number range calculated based on the configured schema start date, index table period and the given schemaEndDate
func (*PeriodConfig) TSDBFormat ¶
func (cfg *PeriodConfig) TSDBFormat() (int, error)
TSDBFormat returns index format corresponding to the `schema` version in the given `PeriodConfig`.
func (*PeriodConfig) UnmarshalYAML ¶
func (cfg *PeriodConfig) UnmarshalYAML(unmarshal func(interface{}) error) error
UnmarshalYAML implements yaml.Unmarshaller.
func (*PeriodConfig) VersionAsInt ¶
func (cfg *PeriodConfig) VersionAsInt() (int, error)
type PeriodicTableConfig ¶
type PeriodicTableConfig struct { Prefix string `yaml:"prefix" doc:"description=Table prefix for all period tables."` Period time.Duration `yaml:"period" doc:"description=Table period."` Tags Tags `yaml:"tags" doc:"description=A map to be added to all managed tables."` }
PeriodicTableConfig is configuration for a set of time-sharded tables.
func (PeriodicTableConfig) MarshalYAML ¶
func (cfg PeriodicTableConfig) MarshalYAML() (interface{}, error)
MarshalYAML implements the yaml.Marshaler interface.
func (*PeriodicTableConfig) PeriodicTables ¶
func (cfg *PeriodicTableConfig) PeriodicTables(from, through model.Time, pCfg ProvisionConfig, beginGrace, endGrace time.Duration, retention time.Duration) []TableDesc
func (*PeriodicTableConfig) TableFor ¶
func (cfg *PeriodicTableConfig) TableFor(t model.Time) string
TableFor calculates the table shard for a given point in time.
func (*PeriodicTableConfig) UnmarshalYAML ¶
func (cfg *PeriodicTableConfig) UnmarshalYAML(unmarshal func(interface{}) error) error
UnmarshalYAML implements the yaml.Unmarshaler interface.
func (PeriodicTableConfig) Validate ¶
func (cfg PeriodicTableConfig) Validate() error
type ProvisionConfig ¶
type ProvisionConfig struct { ActiveTableProvisionConfig `yaml:",inline"` InactiveTableProvisionConfig `yaml:",inline"` }
ProvisionConfig holds config for provisioning capacity for index and chunk tables (on DynamoDB for now)
func (*ProvisionConfig) RegisterFlags ¶
func (cfg *ProvisionConfig) RegisterFlags(argPrefix string, f *flag.FlagSet)
RegisterFlags adds the flags required to config this to the given FlagSet.
type SchemaConfig ¶
type SchemaConfig struct { Configs []PeriodConfig `yaml:"configs"` // contains filtered or unexported fields }
SchemaConfig contains the config for our chunk index schemas
func (SchemaConfig) ChunkTableFor ¶
func (cfg SchemaConfig) ChunkTableFor(t model.Time) (string, error)
ChunkTableFor calculates the chunk table shard for a given point in time.
func (SchemaConfig) ExternalKey ¶
func (cfg SchemaConfig) ExternalKey(ref logproto.ChunkRef) string
Generate the appropriate external key based on cfg.Schema, chunk.Checksum, and chunk.From
func (*SchemaConfig) ForEachAfter ¶
func (cfg *SchemaConfig) ForEachAfter(t model.Time, f func(config *PeriodConfig))
ForEachAfter will call f() on every entry after t, splitting entries if necessary so there is an entry starting at t
func (*SchemaConfig) Load ¶
func (cfg *SchemaConfig) Load() error
Load the yaml file, or build the config from legacy command-line flags
func (*SchemaConfig) RegisterFlags ¶
func (cfg *SchemaConfig) RegisterFlags(f *flag.FlagSet)
RegisterFlags adds the flags required to config this to the given FlagSet.
func (SchemaConfig) SchemaForTime ¶
func (cfg SchemaConfig) SchemaForTime(t model.Time) (PeriodConfig, error)
SchemaForTime returns the Schema PeriodConfig to use for a given point in time.
func (*SchemaConfig) Validate ¶
func (cfg *SchemaConfig) Validate() error
Validate the schema config and returns an error if the validation doesn't pass
func (SchemaConfig) VersionForChunk ¶
func (cfg SchemaConfig) VersionForChunk(ref logproto.ChunkRef) int
VersionForChunk will return the schema version associated with the `From` timestamp of a chunk. The schema and chunk must be valid+compatible as the errors are not checked.
type TableDesc ¶
type TableDesc struct { Name string UseOnDemandIOMode bool ProvisionedRead int64 ProvisionedWrite int64 Tags Tags WriteScale AutoScalingConfig ReadScale AutoScalingConfig }
TableDesc describes a table.
type TableRange ¶
type TableRange struct {
Start, End int64
PeriodConfig *PeriodConfig
}
TableRange represents a range of table numbers built based on the configured schema start/end date and the table period. Both Start and End are inclusive.
func (TableRange) ConfigForTableNumber ¶
func (t TableRange) ConfigForTableNumber(tableNumber int64) *PeriodConfig
func (TableRange) TableInRange ¶
func (t TableRange) TableInRange(tableName string) (bool, error)
TableInRange tells whether given table falls in the range and the tableName has the right prefix based on the schema config.
type TableRanges ¶
type TableRanges []TableRange
TableRanges represents a list of table ranges for multiple schemas.
func GetIndexStoreTableRanges ¶
func GetIndexStoreTableRanges(indexType string, periodicConfigs []PeriodConfig) TableRanges
func (TableRanges) ConfigForTableNumber ¶
func (t TableRanges) ConfigForTableNumber(tableNumber int64) *PeriodConfig
func (TableRanges) TableInRange ¶
func (t TableRanges) TableInRange(tableName string) (bool, error)
TableInRange tells whether given table falls in any of the ranges and the tableName has the right prefix based on the schema config.
func (TableRanges) TableNameFor ¶
func (t TableRanges) TableNameFor(table int64) (string, bool)