config

Documentation

Index

• Constants
• Variables
• func FromEnv(config *Config) error
• func GetConfigOptionGroup() *swag.CommandLineOptionsGroup
• type APIKey
• type AnonymousAccess
• type Authentication
  • func (a Authentication) AnyAuthMethodSelected() bool
  • func (a Authentication) Validate() error
• type Authorization
  • func (a Authorization) Validate() error
• type AutoSchema
  • func (a AutoSchema) Validate() error
• type CORS
• type Config
  • func (c Config) Validate(modProv moduleProvider) error
• type Contextionary
• type DiskUse
  • func (d DiskUse) Validate() error
• type Flags
• type GRPC
• type MemUse
  • func (m MemUse) Validate() error
• type OIDC
• type Persistence
  • func (p Persistence) Validate() error
• type Profiling
• type QueryDefaults
• type Raft
  • func (r *Raft) Validate() error
• type ResourceUsage
  • func (r ResourceUsage) Validate() error
• type WeaviateConfig
  • func (f *WeaviateConfig) GetHostAddress() string
  • func (f *WeaviateConfig) LoadConfig(flags *swag.CommandLineOptionsGroup, logger logrus.FieldLogger) error

Constants

const (
	// These BM25 tuning params can be overwritten on a per-class basis
	DefaultBM25k1 = float32(1.2)
	DefaultBM25b  = float32(0.75)
)

const (
	DefaultMaxImportGoroutinesFactor = float64(1.5)

	DefaultDiskUseWarningPercentage  = uint64(80)
	DefaultDiskUseReadonlyPercentage = uint64(90)
	DefaultMemUseWarningPercentage   = uint64(80)
	// TODO: off by default for now, to make sure
	//       the measurement is reliable. once
	//       confirmed, we can set this to 90
	DefaultMemUseReadonlyPercentage = uint64(0)
)

const (
	DefaultCORSAllowOrigin  = "*"
	DefaultCORSAllowMethods = "*"
	DefaultCORSAllowHeaders = "" /* 486-byte string literal not displayed */
)

const (
	DefaultRaftPort             = 8300
	DefaultRaftInternalPort     = 8301
	DefaultRaftGRPCMaxSize      = 1024 * 1024 * 1024
	DefaultRaftBootstrapTimeout = 90
	DefaultRaftBootstrapExpect  = 1
	DefaultRaftDir              = "raft"
)

const (
	DefaultQueryMaximumResults            = int64(10000)
	DefaultQueryNestedCrossReferenceLimit = int64(100000)
)

const (
	DefaultPersistenceMemtablesFlushDirtyAfter = 60
	DefaultPersistenceMemtablesMaxSize         = 200
	DefaultPersistenceMemtablesMinDuration     = 15
	DefaultPersistenceMemtablesMaxDuration     = 45
	DefaultMaxConcurrentGetRequests            = 0
	DefaultGRPCPort                            = 50051
	DefaultMinimumReplicationFactor            = 1
)

const DefaultCleanupIntervalSeconds = int64(60)

DefaultCleanupIntervalSeconds can be overwritten on a per-class basis

const DefaultConfigFile string = "./weaviate.conf.json"

DefaultConfigFile is the default file when no config file is provided

const DefaultGossipBindPort = 7946

DefaultGossipBindPort uses the hashicorp/memberlist default port value assigned with the use of DefaultLocalConfig

const DefaultPersistenceDataPath string = "./data"

DefaultPersistenceDataPath is the default location for data directory when no location is provided

const DefaultPersistenceHNSWMaxLogSize = 500 * 1024 * 1024 // 500MB for backward compatibility

const DefaultPersistenceLSMMaxSegmentSize = math.MaxInt64

DefaultPersistenceLSMMaxSegmentSize is effectively unlimited for backward compatibility. TODO: consider changing this in a future release and make some noise about it. This is technically a breaking change.

const DefaultQueryDefaultsLimit int64 = 10

DefaultQueryDefaultsLimit is the default query limit when no limit is provided

const VectorizerModuleNone = "none"

const VectorizerModuleText2VecContextionary = "text2vec-contextionary"

TODO: This should be retrieved dynamically from all installed modules

Variables

var DefaultAuthentication = Authentication{
	AnonymousAccess: AnonymousAccess{
		Enabled: true,
	},
}

DefaultAuthentication is the default authentication scheme when no authentication is provided

var GitHash = "unknown"

GitHash keeps the current git hash commit information

var ServerVersion string

ServerVersion is set when the misc handlers are setup. When misc handlers are setup, the entire swagger spec is already being parsed for the server version. This is a good time for us to set ServerVersion, so that the spec only needs to be parsed once.

Functions

func FromEnv

func FromEnv(config *Config) error

FromEnv takes a *Config as it will respect initial config that has been provided by other means (e.g. a config file) and will only extend those that are set

func GetConfigOptionGroup

func GetConfigOptionGroup() *swag.CommandLineOptionsGroup

GetConfigOptionGroup creates an option group for swagger

Types

type APIKey

type APIKey struct {
	Enabled     bool     `json:"enabled" yaml:"enabled"`
	Users       []string `json:"users" yaml:"users"`
	AllowedKeys []string `json:"allowed_keys" yaml:"allowed_keys"`
}

type AnonymousAccess

type AnonymousAccess struct {
	Enabled bool `json:"enabled" yaml:"enabled"`
}

AnonymousAccess considers users without any auth information as authenticated as "anonymous" rather than denying their request immediately. Note that enabling anonymous access ONLY affects Authentication, not Authorization.

type Authentication

type Authentication struct {
	OIDC            OIDC            `json:"oidc" yaml:"oidc"`
	AnonymousAccess AnonymousAccess `json:"anonymous_access" yaml:"anonymous_access"`
	APIKey          APIKey
}

Authentication configuration

func (Authentication) AnyAuthMethodSelected

func (a Authentication) AnyAuthMethodSelected() bool

func (Authentication) Validate

func (a Authentication) Validate() error

Validate the Authentication configuration. This only validates at a general level. Validation specific to the individual auth methods should happen inside their respective packages

type Authorization

type Authorization struct {
	AdminList adminlist.Config `json:"admin_list" yaml:"admin_list"`
}

Authorization configuration

func (Authorization) Validate

func (a Authorization) Validate() error

Validate the Authorization configuration. This only validates at a general level. Validation specific to the individual auth methods should happen inside their respective packages

type AutoSchema

type AutoSchema struct {
	Enabled       bool   `json:"enabled" yaml:"enabled"`
	DefaultString string `json:"defaultString" yaml:"defaultString"`
	DefaultNumber string `json:"defaultNumber" yaml:"defaultNumber"`
	DefaultDate   string `json:"defaultDate" yaml:"defaultDate"`
}

func (AutoSchema) Validate

func (a AutoSchema) Validate() error

type CORS

type CORS struct {
	AllowOrigin  string `json:"allow_origin" yaml:"allow_origin"`
	AllowMethods string `json:"allow_methods" yaml:"allow_methods"`
	AllowHeaders string `json:"allow_headers" yaml:"allow_headers"`
}

type Config

type Config struct {
	Name                                string                   `json:"name" yaml:"name"`
	Debug                               bool                     `json:"debug" yaml:"debug"`
	QueryDefaults                       QueryDefaults            `json:"query_defaults" yaml:"query_defaults"`
	QueryMaximumResults                 int64                    `json:"query_maximum_results" yaml:"query_maximum_results"`
	QueryNestedCrossReferenceLimit      int64                    `json:"query_nested_cross_reference_limit" yaml:"query_nested_cross_reference_limit"`
	Contextionary                       Contextionary            `json:"contextionary" yaml:"contextionary"`
	Authentication                      Authentication           `json:"authentication" yaml:"authentication"`
	Authorization                       Authorization            `json:"authorization" yaml:"authorization"`
	Origin                              string                   `json:"origin" yaml:"origin"`
	Persistence                         Persistence              `json:"persistence" yaml:"persistence"`
	DefaultVectorizerModule             string                   `json:"default_vectorizer_module" yaml:"default_vectorizer_module"`
	DefaultVectorDistanceMetric         string                   `json:"default_vector_distance_metric" yaml:"default_vector_distance_metric"`
	EnableModules                       string                   `json:"enable_modules" yaml:"enable_modules"`
	EnableApiBasedModules               bool                     `json:"enable_api_based_modules" yaml:"enable_api_based_modules"`
	ModulesPath                         string                   `json:"modules_path" yaml:"modules_path"`
	ModuleHttpClientTimeout             time.Duration            `json:"modules_client_timeout" yaml:"modules_client_timeout"`
	AutoSchema                          AutoSchema               `json:"auto_schema" yaml:"auto_schema"`
	Cluster                             cluster.Config           `json:"cluster" yaml:"cluster"`
	Replication                         replication.GlobalConfig `json:"replication" yaml:"replication"`
	Monitoring                          monitoring.Config        `json:"monitoring" yaml:"monitoring"`
	GRPC                                GRPC                     `json:"grpc" yaml:"grpc"`
	Profiling                           Profiling                `json:"profiling" yaml:"profiling"`
	ResourceUsage                       ResourceUsage            `json:"resource_usage" yaml:"resource_usage"`
	MaxImportGoroutinesFactor           float64                  `json:"max_import_goroutine_factor" yaml:"max_import_goroutine_factor"`
	MaximumConcurrentGetRequests        int                      `json:"maximum_concurrent_get_requests" yaml:"maximum_concurrent_get_requests"`
	TrackVectorDimensions               bool                     `json:"track_vector_dimensions" yaml:"track_vector_dimensions"`
	ReindexVectorDimensionsAtStartup    bool                     `json:"reindex_vector_dimensions_at_startup" yaml:"reindex_vector_dimensions_at_startup"`
	DisableLazyLoadShards               bool                     `json:"disable_lazy_load_shards" yaml:"disable_lazy_load_shards"`
	ForceFullReplicasSearch             bool                     `json:"force_full_replicas_search" yaml:"force_full_replicas_search"`
	RecountPropertiesAtStartup          bool                     `json:"recount_properties_at_startup" yaml:"recount_properties_at_startup"`
	ReindexSetToRoaringsetAtStartup     bool                     `json:"reindex_set_to_roaringset_at_startup" yaml:"reindex_set_to_roaringset_at_startup"`
	IndexMissingTextFilterableAtStartup bool                     `json:"index_missing_text_filterable_at_startup" yaml:"index_missing_text_filterable_at_startup"`
	DisableGraphQL                      bool                     `json:"disable_graphql" yaml:"disable_graphql"`
	AvoidMmap                           bool                     `json:"avoid_mmap" yaml:"avoid_mmap"`
	CORS                                CORS                     `json:"cors" yaml:"cors"`
	DisableTelemetry                    bool                     `json:"disable_telemetry" yaml:"disable_telemetry"`
	HNSWStartupWaitForVectorCache       bool                     `json:"hnsw_startup_wait_for_vector_cache" yaml:"hnsw_startup_wait_for_vector_cache"`
	Sentry                              *entsentry.ConfigOpts    `json:"sentry" yaml:"sentry"`

	// Raft Specific configuration
	// TODO-RAFT: Do we want to be able to specify these with config file as well ?
	Raft Raft
}

Config outline of the config file

func (Config) Validate

func (c Config) Validate(modProv moduleProvider) error

Validate the non-nested parameters. Nested objects must provide their own validation methods

type Contextionary

type Contextionary struct {
	URL string `json:"url" yaml:"url"`
}

type DiskUse

type DiskUse struct {
	WarningPercentage  uint64 `json:"warning_percentage" yaml:"warning_percentage"`
	ReadOnlyPercentage uint64 `json:"readonly_percentage" yaml:"readonly_percentage"`
}

func (DiskUse) Validate

func (d DiskUse) Validate() error

type Flags

type Flags struct {
	ConfigFile string `long:"config-file" description:"path to config file (default: ./weaviate.conf.json)"`

	RaftPort               int      `long:"raft-port" description:"the port used by Raft for inter-node communication"`
	RaftInternalRPCPort    int      `long:"raft-internal-rpc-port" description:"the port used for internal RPCs within the cluster"`
	RaftRPCMessageMaxSize  int      `long:"raft-rpc-message-max-size" description:"maximum internal raft grpc message size in bytes, defaults to 1073741824"`
	RaftJoin               []string `` /* 240-byte string literal not displayed */
	RaftBootstrapTimeout   int      `` /* 148-byte string literal not displayed */
	RaftBootstrapExpect    int      `long:"raft-bootstrap-expect" description:"specifies the number of server nodes to wait for before bootstrapping the cluster"`
	RaftHeartbeatTimeout   int      `long:"raft-heartbeat-timeout" description:"raft heartbeat timeout"`
	RaftElectionTimeout    int      `long:"raft-election-timeout" description:"raft election timeout"`
	RaftSnapshotThreshold  int      `long:"raft-snap-threshold" description:"number of outstanding log entries before performing a snapshot"`
	RaftSnapshotInterval   int      `long:"raft-snap-interval" description:"controls how often raft checks if it should perform a snapshot"`
	RaftMetadataOnlyVoters bool     `` /* 130-byte string literal not displayed */
}

Flags are input options

type GRPC

type GRPC struct {
	Port     int    `json:"port" yaml:"port"`
	CertFile string `json:"certFile" yaml:"certFile"`
	KeyFile  string `json:"keyFile" yaml:"keyFile"`
}

Support independent TLS credentials for gRPC

type MemUse

type MemUse struct {
	WarningPercentage  uint64 `json:"warning_percentage" yaml:"warning_percentage"`
	ReadOnlyPercentage uint64 `json:"readonly_percentage" yaml:"readonly_percentage"`
}

func (MemUse) Validate

func (m MemUse) Validate() error

type OIDC

type OIDC struct {
	Enabled           bool     `json:"enabled" yaml:"enabled"`
	Issuer            string   `json:"issuer" yaml:"issuer"`
	ClientID          string   `json:"client_id" yaml:"client_id"`
	SkipClientIDCheck bool     `yaml:"skip_client_id_check" json:"skip_client_id_check"`
	UsernameClaim     string   `yaml:"username_claim" json:"username_claim"`
	GroupsClaim       string   `yaml:"groups_claim" json:"groups_claim"`
	Scopes            []string `yaml:"scopes" json:"scopes"`
}

OIDC configures the OIDC middleware

type Persistence

type Persistence struct {
	DataPath                          string `json:"dataPath" yaml:"dataPath"`
	MemtablesFlushDirtyAfter          int    `json:"flushDirtyMemtablesAfter" yaml:"flushDirtyMemtablesAfter"`
	MemtablesMaxSizeMB                int    `json:"memtablesMaxSizeMB" yaml:"memtablesMaxSizeMB"`
	MemtablesMinActiveDurationSeconds int    `json:"memtablesMinActiveDurationSeconds" yaml:"memtablesMinActiveDurationSeconds"`
	MemtablesMaxActiveDurationSeconds int    `json:"memtablesMaxActiveDurationSeconds" yaml:"memtablesMaxActiveDurationSeconds"`
	LSMMaxSegmentSize                 int64  `json:"lsmMaxSegmentSize" yaml:"lsmMaxSegmentSize"`
	HNSWMaxLogSize                    int64  `json:"hnswMaxLogSize" yaml:"hnswMaxLogSize"`
}

func (Persistence) Validate

func (p Persistence) Validate() error

type Profiling

type Profiling struct {
	BlockProfileRate     int  `json:"blockProfileRate" yaml:"blockProfileRate"`
	MutexProfileFraction int  `json:"mutexProfileFraction" yaml:"mutexProfileFraction"`
	Disabled             bool `json:"disabled" yaml:"disabled"`
	Port                 int  `json:"port" yaml:"port"`
}

type QueryDefaults

type QueryDefaults struct {
	Limit int64 `json:"limit" yaml:"limit"`
}

QueryDefaults for optional parameters

type Raft

type Raft struct {
	Port                   int
	InternalRPCPort        int
	RPCMessageMaxSize      int
	Join                   []string
	SnapshotThreshold      uint64
	HeartbeatTimeout       time.Duration
	ElectionTimeout        time.Duration
	SnapshotInterval       time.Duration
	ConsistencyWaitTimeout time.Duration

	BootstrapTimeout   time.Duration
	BootstrapExpect    int
	MetadataOnlyVoters bool
}

func (*Raft) Validate

func (r *Raft) Validate() error

type ResourceUsage

type ResourceUsage struct {
	DiskUse DiskUse
	MemUse  MemUse
}

func (ResourceUsage) Validate

func (r ResourceUsage) Validate() error

type WeaviateConfig

type WeaviateConfig struct {
	Config   Config
	Hostname string
	Scheme   string
}

WeaviateConfig represents the used schema's

func (*WeaviateConfig) GetHostAddress

func (f *WeaviateConfig) GetHostAddress() string

GetHostAddress from config locations

func (*WeaviateConfig) LoadConfig

func (f *WeaviateConfig) LoadConfig(flags *swag.CommandLineOptionsGroup, logger logrus.FieldLogger) error

LoadConfig from config locations. The load order for configuration values if the following 1. Config file 2. Environment variables 3. Command line flags If a config option is specified multiple times in different locations, the latest one will be used in this order.