config

package
v1.16.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Apr 3, 2024 License: Apache-2.0 Imports: 12 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	UseLocalConfiguration   = "local-settings"
	QuickstartConfiguration = "quickstart"
)

UseLocalConfiguration set to true will add defaults that enable a lakeFS run without any other configuration like DB or blockstore.

View Source 
const (
	AuthRBACSimplified = "simplified"
	AuthRBACExternal   = "external"
	AuthRBACInternal   = "internal"
)
View Source 
const (
	DefaultListenAddress             = "0.0.0.0:8000"
	DefaultLoggingLevel              = "INFO"
	DefaultLoggingAuditLogLevel      = "DEBUG"
	BlockstoreTypeKey                = "blockstore.type"
	DefaultQuickstartUsername        = "quickstart"
	DefaultQuickstartKeyID           = "AKIAIOSFOLQUICKSTART"                     //nolint:gosec
	DefaultQuickstartSecretKey       = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" // nolint:gosec
	DefaultAuthAPIHealthCheckTimeout = 20 * time.Second
)
View Source 
const (
	FieldMaskedValue   = "******"
	FieldMaskedNoValue = "------"
)

Variables

View Source 
var (
	ErrBadConfiguration    = errors.New("bad configuration")
	ErrBadDomainNames      = fmt.Errorf("%w: domain names are prefixes", ErrBadConfiguration)
	ErrMissingRequiredKeys = fmt.Errorf("%w: missing required keys", ErrBadConfiguration)
)
View Source 
var (
	ErrMustBeString = errors.New("must be a string")
)

Functions

func DecodeOnlyString added in v0.65.0 

func DecodeOnlyString(fromValue reflect.Value, toValue reflect.Value) (interface{}, error)

DecodeOnlyString is a mapstructure.HookFuncType that decodes a string value as an OnlyString, but fails on all other values. It is useful to force parsing of a field that can contain just digits as a string, when the leading digit might be 0.

func DecodeStrings added in v0.40.0 

func DecodeStrings(fromValue reflect.Value, toValue reflect.Value) (interface{}, error)

DecodeStrings is a mapstructure.HookFuncType that decodes a single string value or a slice of strings into Strings.

func GetStructKeys added in v0.40.0 

func GetStructKeys(typ reflect.Type, tag, squashValue string) []string

GetStructKeys returns all keys in a nested struct type, taking the name from the tag name or the field name. It handles an additional suffix squashValue like mapstructure does: if present on an embedded struct, name components for that embedded struct should not be included. It does not handle maps, does chase pointers, but does not check for loops in nesting.

func MapLoggingFields added in v0.50.0 

func MapLoggingFields(value interface{}) logging.Fields

MapLoggingFields returns all logging.Fields formatted based on our configuration keys 'dot.name.key' with associated values. Supports squash, and secret to skip printing out secrets.

func Unmarshal added in v0.90.0 

func Unmarshal(c *Config) error

func ValidateMissingRequiredKeys added in v0.48.0 

func ValidateMissingRequiredKeys(value interface{}, tag, squashValue string) []string

ValidateMissingRequiredKeys returns all keys of value in GetStructKeys format that have an additional required tag set but are unset.

Types

type Config 

type Config struct {
	ListenAddress string `mapstructure:"listen_address"`
	TLS           struct {
		Enabled  bool   `mapstructure:"enabled"`
		CertFile string `mapstructure:"cert_file"`
		KeyFile  string `mapstructure:"key_file"`
	} `mapstructure:"tls"`

	Actions struct {
		// ActionsEnabled set to false will block any hook execution
		Enabled bool `mapstructure:"enabled"`
		Lua     struct {
			NetHTTPEnabled bool `mapstructure:"net_http_enabled"`
		} `mapstructure:"lua"`
		Env struct {
			Enabled bool   `mapstructure:"enabled"`
			Prefix  string `mapstructure:"prefix"`
		} `mapstructure:"env"`
	} `mapstructure:"actions"`

	Logging struct {
		Format        string   `mapstructure:"format"`
		Level         string   `mapstructure:"level"`
		Output        []string `mapstructure:"output"`
		FileMaxSizeMB int      `mapstructure:"file_max_size_mb"`
		FilesKeep     int      `mapstructure:"files_keep"`
		AuditLogLevel string   `mapstructure:"audit_log_level"`
		// TraceRequestHeaders work only on 'trace' level, default is false as it may log sensitive data to the log
		TraceRequestHeaders bool `mapstructure:"trace_request_headers"`
	}

	Database struct {
		// DropTables Development flag to delete tables after successful migration to KV
		DropTables bool `mapstructure:"drop_tables"`
		// Type Name of the KV Store driver DB implementation which is available according to the kv package Drivers function
		Type string `mapstructure:"type" validate:"required"`

		Local *struct {
			// Path - Local directory path to store the DB files
			Path string `mapstructure:"path"`
			// SyncWrites - Sync ensures data written to disk on each write instead of mem cache
			SyncWrites bool `mapstructure:"sync_writes"`
			// PrefetchSize - Number of elements to prefetch while iterating
			PrefetchSize int `mapstructure:"prefetch_size"`
			// EnableLogging - Enable store and badger (trace only) logging
			EnableLogging bool `mapstructure:"enable_logging"`
		} `mapstructure:"local"`

		Postgres *struct {
			ConnectionString      SecureString  `mapstructure:"connection_string"`
			MaxOpenConnections    int32         `mapstructure:"max_open_connections"`
			MaxIdleConnections    int32         `mapstructure:"max_idle_connections"`
			ConnectionMaxLifetime time.Duration `mapstructure:"connection_max_lifetime"`
			ScanPageSize          int           `mapstructure:"scan_page_size"`
			Metrics               bool          `mapstructure:"metrics"`
		}

		DynamoDB *struct {
			// The name of the DynamoDB table to be used as KV
			TableName string `mapstructure:"table_name"`

			// Maximal number of items per page during scan operation
			ScanLimit int64 `mapstructure:"scan_limit"`

			// The endpoint URL of the DynamoDB endpoint
			// Can be used to redirect to DynamoDB on AWS, local docker etc.
			Endpoint string `mapstructure:"endpoint"`

			// AWS connection details - region and credentials
			// This will override any such details that are already exist in the system
			// While in general, AWS region and credentials are configured in the system for AWS usage,
			// these can be used to specify fake values, that cna be used to connect to local DynamoDB,
			// in case there are no credentials configured in the system
			// This is a client requirement as described in section 4 in
			// https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/DynamoDBLocal.DownloadingAndRunning.html
			AwsRegion          string       `mapstructure:"aws_region"`
			AwsProfile         string       `mapstructure:"aws_profile"`
			AwsAccessKeyID     SecureString `mapstructure:"aws_access_key_id"`
			AwsSecretAccessKey SecureString `mapstructure:"aws_secret_access_key"`

			// HealthCheckInterval - Interval to run health check for the DynamoDB instance
			// Won't run when is equal or less than 0.
			HealthCheckInterval time.Duration `mapstructure:"health_check_interval"`

			// MaxAttempts - Specifies the maximum number attempts to make on a request.
			MaxAttempts int `mapstructure:"max_attempts"`
		} `mapstructure:"dynamodb"`

		CosmosDB *struct {
			Key        SecureString `mapstructure:"key"`
			Endpoint   string       `mapstructure:"endpoint"`
			Database   string       `mapstructure:"database"`
			Container  string       `mapstructure:"container"`
			Throughput int32        `mapstructure:"throughput"`
			Autoscale  bool         `mapstructure:"autoscale"`
		} `mapstructure:"cosmosdb"`
	}

	Auth struct {
		Cache struct {
			Enabled bool          `mapstructure:"enabled"`
			Size    int           `mapstructure:"size"`
			TTL     time.Duration `mapstructure:"ttl"`
			Jitter  time.Duration `mapstructure:"jitter"`
		} `mapstructure:"cache"`
		Encrypt struct {
			SecretKey SecureString `mapstructure:"secret_key" validate:"required"`
		} `mapstructure:"encrypt"`
		API struct {
			// Endpoint for authorization operations
			Endpoint           string        `mapstructure:"endpoint"`
			Token              SecureString  `mapstructure:"token"`
			SupportsInvites    bool          `mapstructure:"supports_invites"`
			HealthCheckTimeout time.Duration `mapstructure:"health_check_timeout"`
			SkipHealthCheck    bool          `mapstructure:"skip_health_check"`
		} `mapstructure:"api"`
		AuthenticationAPI struct {
			// Endpoint for authentication operations
			Endpoint string `mapstructure:"endpoint"`
		} `mapstructure:"authentication_api"`
		RemoteAuthenticator struct {
			// Enabled if set true will enable remote authentication
			Enabled bool `mapstructure:"enabled"`
			// ExternalPrincipalAuth configuration related external principals
			ExternalPrincipalsEnabled bool `mapstructure:"external_principals_enabled"`
			// Endpoint URL of the remote authentication service (e.g. https://my-auth.example.com/auth)
			Endpoint string `mapstructure:"endpoint"`
			// DefaultUserGroup is the default group for the users authenticated by the remote service
			DefaultUserGroup string `mapstructure:"default_user_group"`
			// RequestTimeout timeout for remote authentication requests
			RequestTimeout time.Duration `mapstructure:"request_timeout"`
		} `mapstructure:"remote_authenticator"`
		OIDC                   OIDC                   `mapstructure:"oidc"`
		CookieAuthVerification CookieAuthVerification `mapstructure:"cookie_auth_verification"`
		// LogoutRedirectURL is the URL on which to mount the
		// server-side logout.
		LogoutRedirectURL string        `mapstructure:"logout_redirect_url"`
		LoginDuration     time.Duration `mapstructure:"login_duration"`
		UIConfig          struct {
			RBAC               string   `mapstructure:"rbac"`
			LoginURL           string   `mapstructure:"login_url"`
			LoginFailedMessage string   `mapstructure:"login_failed_message"`
			FallbackLoginURL   *string  `mapstructure:"fallback_login_url"`
			FallbackLoginLabel *string  `mapstructure:"fallback_login_label"`
			LoginCookieNames   []string `mapstructure:"login_cookie_names"`
			LogoutURL          string   `mapstructure:"logout_url"`
		} `mapstructure:"ui_config"`
	} `mapstructure:"auth"`
	Blockstore struct {
		Type                   string  `mapstructure:"type" validate:"required"`
		DefaultNamespacePrefix *string `mapstructure:"default_namespace_prefix"`
		Local                  *struct {
			Path                    string   `mapstructure:"path"`
			ImportEnabled           bool     `mapstructure:"import_enabled"`
			ImportHidden            bool     `mapstructure:"import_hidden"`
			AllowedExternalPrefixes []string `mapstructure:"allowed_external_prefixes"`
		} `mapstructure:"local"`
		S3 *struct {
			S3AuthInfo                    `mapstructure:",squash"`
			Region                        string        `mapstructure:"region"`
			Endpoint                      string        `mapstructure:"endpoint"`
			MaxRetries                    int           `mapstructure:"max_retries"`
			ForcePathStyle                bool          `mapstructure:"force_path_style"`
			DiscoverBucketRegion          bool          `mapstructure:"discover_bucket_region"`
			SkipVerifyCertificateTestOnly bool          `mapstructure:"skip_verify_certificate_test_only"`
			ServerSideEncryption          string        `mapstructure:"server_side_encryption"`
			ServerSideEncryptionKmsKeyID  string        `mapstructure:"server_side_encryption_kms_key_id"`
			PreSignedExpiry               time.Duration `mapstructure:"pre_signed_expiry"`
			DisablePreSigned              bool          `mapstructure:"disable_pre_signed"`
			DisablePreSignedUI            bool          `mapstructure:"disable_pre_signed_ui"`
			DisablePreSignedMultipart     bool          `mapstructure:"disable_pre_signed_multipart"`
			ClientLogRetries              bool          `mapstructure:"client_log_retries"`
			ClientLogRequest              bool          `mapstructure:"client_log_request"`
			WebIdentity                   *struct {
				SessionDuration     time.Duration `mapstructure:"session_duration"`
				SessionExpiryWindow time.Duration `mapstructure:"session_expiry_window"`
			} `mapstructure:"web_identity"`
		} `mapstructure:"s3"`
		Azure *struct {
			TryTimeout       time.Duration `mapstructure:"try_timeout"`
			StorageAccount   string        `mapstructure:"storage_account"`
			StorageAccessKey string        `mapstructure:"storage_access_key"`
			// Deprecated: Value ignored
			AuthMethod         string        `mapstructure:"auth_method"`
			PreSignedExpiry    time.Duration `mapstructure:"pre_signed_expiry"`
			DisablePreSigned   bool          `mapstructure:"disable_pre_signed"`
			DisablePreSignedUI bool          `mapstructure:"disable_pre_signed_ui"`
			ChinaCloud         bool          `mapstructure:"china_cloud"`
			// TestEndpointURL for testing purposes
			TestEndpointURL string `mapstructure:"test_endpoint_url"`
		} `mapstructure:"azure"`
		GS *struct {
			S3Endpoint         string        `mapstructure:"s3_endpoint"`
			CredentialsFile    string        `mapstructure:"credentials_file"`
			CredentialsJSON    string        `mapstructure:"credentials_json"`
			PreSignedExpiry    time.Duration `mapstructure:"pre_signed_expiry"`
			DisablePreSigned   bool          `mapstructure:"disable_pre_signed"`
			DisablePreSignedUI bool          `mapstructure:"disable_pre_signed_ui"`
		} `mapstructure:"gs"`
	} `mapstructure:"blockstore"`
	Committed struct {
		LocalCache struct {
			SizeBytes             int64   `mapstructure:"size_bytes"`
			Dir                   string  `mapstructure:"dir"`
			MaxUploadersPerWriter int     `mapstructure:"max_uploaders_per_writer"`
			RangeProportion       float64 `mapstructure:"range_proportion"`
			MetaRangeProportion   float64 `mapstructure:"metarange_proportion"`
		} `mapstructure:"local_cache"`
		BlockStoragePrefix string `mapstructure:"block_storage_prefix"`
		Permanent          struct {
			MinRangeSizeBytes      uint64  `mapstructure:"min_range_size_bytes"`
			MaxRangeSizeBytes      uint64  `mapstructure:"max_range_size_bytes"`
			RangeRaggednessEntries float64 `mapstructure:"range_raggedness_entries"`
		} `mapstructure:"permanent"`
		SSTable struct {
			Memory struct {
				CacheSizeBytes int64 `mapstructure:"cache_size_bytes"`
			} `mapstructure:"memory"`
		} `mapstructure:"sstable"`
	} `mapstructure:"committed"`
	UGC struct {
		PrepareMaxFileSize int64         `mapstructure:"prepare_max_file_size"`
		PrepareInterval    time.Duration `mapstructure:"prepare_interval"`
	} `mapstructure:"ugc"`
	Graveler struct {
		EnsureReadableRootNamespace bool `mapstructure:"ensure_readable_root_namespace"`
		BatchDBIOTransactionMarkers bool `mapstructure:"batch_dbio_transaction_markers"`
		CompactionSensorThreshold   int  `mapstructure:"compaction_sensor_threshold"`
		RepositoryCache             struct {
			Size   int           `mapstructure:"size"`
			Expiry time.Duration `mapstructure:"expiry"`
			Jitter time.Duration `mapstructure:"jitter"`
		} `mapstructure:"repository_cache"`
		CommitCache struct {
			Size   int           `mapstructure:"size"`
			Expiry time.Duration `mapstructure:"expiry"`
			Jitter time.Duration `mapstructure:"jitter"`
		} `mapstructure:"commit_cache"`
		Background struct {
			RateLimit int `mapstructure:"rate_limit"`
		} `mapstructure:"background"`
	} `mapstructure:"graveler"`
	Gateways struct {
		S3 struct {
			DomainNames       Strings `mapstructure:"domain_name"`
			Region            string  `mapstructure:"region"`
			FallbackURL       string  `mapstructure:"fallback_url"`
			VerifyUnsupported bool    `mapstructure:"verify_unsupported"`
		} `mapstructure:"s3"`
	}
	Stats struct {
		Enabled       bool          `mapstructure:"enabled"`
		Address       string        `mapstructure:"address"`
		FlushInterval time.Duration `mapstructure:"flush_interval"`
		FlushSize     int           `mapstructure:"flush_size"`
		Extended      bool          `mapstructure:"extended"`
	} `mapstructure:"stats"`
	EmailSubscription struct {
		Enabled bool `mapstructure:"enabled"`
	} `mapstructure:"email_subscription"`
	Installation struct {
		FixedID         string       `mapstructure:"fixed_id"`
		UserName        string       `mapstructure:"user_name"`
		AccessKeyID     SecureString `mapstructure:"access_key_id"`
		SecretAccessKey SecureString `mapstructure:"secret_access_key"`
	} `mapstructure:"installation"`
	Security struct {
		CheckLatestVersion      bool          `mapstructure:"check_latest_version"`
		CheckLatestVersionCache time.Duration `mapstructure:"check_latest_version_cache"`
		AuditCheckInterval      time.Duration `mapstructure:"audit_check_interval"`
		AuditCheckURL           string        `mapstructure:"audit_check_url"`
	} `mapstructure:"security"`
	UI struct {
		// Enabled - control serving of embedded UI
		Enabled  bool `mapstructure:"enabled"`
		Snippets []struct {
			ID   string `mapstructure:"id"`
			Code string `mapstructure:"code"`
		} `mapstructure:"snippets"`
	} `mapstructure:"ui"`
	UsageReport struct {
		Enabled       bool          `mapstructure:"enabled"`
		FlushInterval time.Duration `mapstructure:"flush_interval"`
	} `mapstructure:"usage_report"`
}

Config - Output struct of configuration, used to validate. If you read a key using a viper accessor rather than accessing a field of this struct, that key will *not* be validated. So don't do that.

func NewConfig 

func NewConfig(cfgType string) (*Config, error)

func (*Config) BlockstoreAzureParams added in v0.89.0 

func (c *Config) BlockstoreAzureParams() (blockparams.Azure, error)

func (*Config) BlockstoreGSParams added in v0.89.0 

func (c *Config) BlockstoreGSParams() (blockparams.GS, error)

func (*Config) BlockstoreLocalParams added in v0.89.0 

func (c *Config) BlockstoreLocalParams() (blockparams.Local, error)

func (*Config) BlockstoreS3Params added in v0.89.0 

func (c *Config) BlockstoreS3Params() (blockparams.S3, error)

func (*Config) BlockstoreType added in v0.89.0 

func (c *Config) BlockstoreType() string

func (*Config) IsAuthTypeAPI added in v0.63.0 

func (c *Config) IsAuthTypeAPI() bool

func (*Config) IsAuthUISimplified added in v0.99.0 

func (c *Config) IsAuthUISimplified() bool

func (*Config) IsAuthenticationTypeAPI added in v1.15.0 

func (c *Config) IsAuthenticationTypeAPI() bool

func (*Config) IsExternalPrincipalsEnabled added in v1.14.0 

func (c *Config) IsExternalPrincipalsEnabled() bool

func (*Config) UISnippets added in v0.89.0 

func (c *Config) UISnippets() []apiparams.CodeSnippet

func (*Config) Validate added in v0.48.0 

func (c *Config) Validate() error

type CookieAuthVerification added in v0.95.0 

type CookieAuthVerification struct {
	// ValidateIDTokenClaims if set will validate the values (e.g., department: "R&D") exist in the token claims
	ValidateIDTokenClaims map[string]string `mapstructure:"validate_id_token_claims"`
	// DefaultInitialGroups is a list of groups to add to the user on the lakeFS side
	DefaultInitialGroups []string `mapstructure:"default_initial_groups"`
	// InitialGroupsClaimName comma separated list of groups to add to the user on the lakeFS side
	InitialGroupsClaimName string `mapstructure:"initial_groups_claim_name"`
	// FriendlyNameClaimName is the claim name to use as the user's friendly name in places like the UI
	FriendlyNameClaimName string `mapstructure:"friendly_name_claim_name"`
	// ExternalUserIDClaimName is the claim name to use as the user identifier with an IDP
	ExternalUserIDClaimName string `mapstructure:"external_user_id_claim_name"`
	// AuthSource tag each user with label of the IDP
	AuthSource string `mapstructure:"auth_source"`
	// PersistFriendlyName should we persist the friendly name in the KV store
	PersistFriendlyName bool `mapstructure:"persist_friendly_name"`
}

CookieAuthVerification is related to auth based on a cookie set by an external service TODO(isan) consolidate with OIDC

type OIDC added in v0.69.0 

type OIDC struct {
	// configure how users are handled on the lakeFS side:
	ValidateIDTokenClaims  map[string]string `mapstructure:"validate_id_token_claims"`
	DefaultInitialGroups   []string          `mapstructure:"default_initial_groups"`
	InitialGroupsClaimName string            `mapstructure:"initial_groups_claim_name"`
	FriendlyNameClaimName  string            `mapstructure:"friendly_name_claim_name"`
	PersistFriendlyName    bool              `mapstructure:"persist_friendly_name"`
}

type OnlyString added in v0.65.0 

type OnlyString string

OnlyString is a string that can deserialize only from a string. Use it to prevent YAML configuration reading a number-like string with leading zeros, and then Viper using mapstructure to convert it silently back to a string and losing the leading zeros.

func (OnlyString) String added in v0.107.0 

func (o OnlyString) String() string

type S3AuthInfo added in v0.40.0 

type S3AuthInfo struct {
	CredentialsFile string `mapstructure:"credentials_file"`
	Profile         string
	Credentials     *struct {
		AccessKeyID     SecureString `mapstructure:"access_key_id"`
		SecretAccessKey SecureString `mapstructure:"secret_access_key"`
		SessionToken    SecureString `mapstructure:"session_token"`
	}
}

S3AuthInfo holds S3-style authentication.

type SecureString added in v0.50.0 

type SecureString string

func (SecureString) SecureValue added in v0.62.0 

func (s SecureString) SecureValue() string

SecureValue returns the actual value of s as a string.

func (SecureString) String added in v0.50.0 

func (SecureString) String() string

String returns an elided version. It is safe to call for logging.

type Strings added in v0.40.0 

type Strings []string

Strings is a []string that mapstructure can deserialize from a single string or from a list of strings.

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.