helper

Documentation

Index

• Constants
• Variables
• func CopiedBytes(source []byte) (destination []byte)
• func EtagWithInternalState(md5 string, md5Writer hash.Hash) string
• func FileExists(path string) bool
• func Filter(xs []string, f func(string) bool) []string
• func GenerateRandomId() []byte
• func HasBucketInDomain(host string, domains []string) (ok bool, bucket string)
• func Keys(v interface{}) []string
• func Map(xs []string, f func(string) string) []string
• func MarshalTOMLConfig() error
• func Md5FromEtag(etag string) string
• func Md5WriterFromEtag(etag string) (hash.Hash, error)
• func MsgPackMarshal(v interface{}) ([]byte, error)
• func MsgPackUnMarshal(data []byte, v interface{}) error
• func PanicOnError(err error, message string)
• func ReadJsonBody(body io.ReadCloser, out interface{}) (err error)
• func ReusePortListener(host, port string) (listener net.Listener, err error)
• func SetupConfig()
• func StringInSlice(s string, ss []string) bool
• func SubString(str string, begin, length int) (substr string)
• func Ternary(IF bool, THEN interface{}, ELSE interface{}) interface{}
• func UnicodeIndex(str, substr string) int
• type Config
• type PluginConfig

Constants

const (
	YIG_CONF_PATH         = "/etc/yig/yig.toml"
	MIN_BUFFER_SIZE int64 = 512 << 10 // 512k
	MAX_BUFEER_SIZE int64 = 8 << 20   // 8M

	DEFAULTLOCK    = 45 // 45min
	DEFAULTREFRESH = 30 // 30 min

)

Variables

var AccessLogger log.Logger

var ErrorLogger log.Logger

var Logger log.Logger

Global singleton loggers

Functions

func CopiedBytes

func CopiedBytes(source []byte) (destination []byte)

func EtagWithInternalState

func EtagWithInternalState(md5 string, md5Writer hash.Hash) string

func FileExists

func FileExists(path string) bool

func Filter

func Filter(xs []string, f func(string) bool) []string

func GenerateRandomId

func GenerateRandomId() []byte

func HasBucketInDomain

func HasBucketInDomain(host string, domains []string) (ok bool, bucket string)

func Keys

func Keys(v interface{}) []string

Get keys of a map, i.e. map[string]interface{} -> []string Note that some type checks are omitted for efficiency, you need to ensure them yourself, otherwise your program should panic

func Map

func Map(xs []string, f func(string) string) []string

func MarshalTOMLConfig

func MarshalTOMLConfig() error

func Md5FromEtag

func Md5FromEtag(etag string) string

func Md5WriterFromEtag

func Md5WriterFromEtag(etag string) (hash.Hash, error)

For appendable upload, we need to calculate the whole file md5 incrementally, so we hack "etag" field in db to store md5 calculator internal state. Format if state is stored: <md5 hex>,<internal state hex>

func MsgPackMarshal

func MsgPackMarshal(v interface{}) ([]byte, error)

func MsgPackUnMarshal

func MsgPackUnMarshal(data []byte, v interface{}) error

func PanicOnError

func PanicOnError(err error, message string)

func ReadJsonBody

func ReadJsonBody(body io.ReadCloser, out interface{}) (err error)

read from ReadCloser and unmarshal to out; `out` should be of POINTER type

func ReusePortListener

func ReusePortListener(host, port string) (listener net.Listener, err error)

set "SO_REUSEADDR" and "SO_REUSEPORT" on socket option and listen

func SetupConfig

func SetupConfig()

func StringInSlice

func StringInSlice(s string, ss []string) bool

func SubString

func SubString(str string, begin, length int) (substr string)

func Ternary

func Ternary(IF bool, THEN interface{}, ELSE interface{}) interface{}

mimic `?:` operator Need type assertion to convert output to expected type

func UnicodeIndex

func UnicodeIndex(str, substr string) int

Types

type Config

type Config struct {
	Brand                []string                `toml:"brand"`    // brand name that YIG supported
	S3Domain             []string                `toml:"s3domain"` // Domain name of YIG
	Region               string                  `toml:"region"`   // Region name this instance belongs to, e.g cn-bj-1
	Plugins              map[string]PluginConfig `toml:"plugins"`
	PiggybackUpdateUsage bool                    `toml:"piggyback_update_usage"`
	LogPath              string                  `toml:"log_path"`
	AccessLogPath        string                  `toml:"access_log_path"`
	AccessLogFormat      string                  `toml:"access_log_format"`
	ErrorLogPath         string                  `toml:"error_log_path"`
	PanicLogPath         string                  `toml:"panic_log_path"`
	PidFile              string                  `toml:"pid_file"`
	BindApiAddress       string                  `toml:"api_listener"`
	BindAdminAddress     string                  `toml:"admin_listener"`
	SSLKeyPath           string                  `toml:"ssl_key_path"`
	SSLCertPath          string                  `toml:"ssl_cert_path"`

	InstanceId             string // if empty, generated one at server startup
	ConcurrentRequestLimit int

	MgThread         int `toml:"mg_thread"`
	MgScanInterval   int `toml:"mg_scan_interval"`
	MgObjectCooldown int `toml:"mg_object_cooldown"`

	DebugMode         bool     `toml:"debug_mode"`
	EnablePProf       bool     `toml:"enable_pprof"`
	BindPProfAddress  string   `toml:"pprof_listener"`
	AdminKey          string   `toml:"admin_key"` //used for tools/admin to communicate with yig
	LcThread          int      //used for tools/lc only, set worker numbers to do lc
	LogLevel          string   `toml:"log_level"` // "info", "warn", "error"
	CephConfigPattern string   `toml:"ceph_config_pattern"`
	ReservedOrigins   string   `toml:"reserved_origins"` // www.ccc.com,www.bbb.com,127.0.0.1
	MetaStore         string   `toml:"meta_store"`
	TidbInfo          string   `toml:"tidb_info"`
	PdAddress         []string `toml:"pd_address"`
	KeepAlive         bool     `toml:"keepalive"`
	EnableCompression bool     `toml:"enable_compression"`
	KafkaBrokers      []string `toml:"kafka_brokers"` // list of `IP:Port`
	GcTopic           string   `toml:"gc_topic"`

	LifecycleSpec string `toml:"lifecycle_spec"` // use for Lifecycle timing

	//About cache
	EnableUsagePush       bool     `toml:"enable_usage_push"`
	RedisStore            string   `toml:"redis_store"`   // Choose redis connection method
	RedisAddress          string   `toml:"redis_address"` // redis connection string, e.g localhost:1234
	RedisGroup            []string `toml:"redis_group"`
	RedisQosGroup         []string `toml:"redis_qos_group"`
	RedisConnectionNumber int      `toml:"redis_connection_number"` // number of connections to redis(i.e max concurrent request number)
	RedisPassword         string   `toml:"redis_password"`          // redis auth password
	MetaCacheType         int      `toml:"meta_cache_type"`
	MetaCacheTTL          int      `toml:"meta_cache_ttl"` //ttl second
	EnableDataCache       bool     `toml:"enable_data_cache"`
	RedisMaxRetries       int      `toml:"redis_max_retries"`
	RedisConnectTimeout   int      `toml:"redis_connect_timeout"`
	RedisReadTimeout      int      `toml:"redis_read_timeout"`
	RedisWriteTimeout     int      `toml:"redis_write_timeout"`
	RedisKeepAlive        int      `toml:"redis_keepalive"`
	RedisPoolMaxIdle      int      `toml:"redis_pool_max_idle"`
	RedisPoolIdleTimeout  int      `toml:"redis_pool_idle_timeout"`
	RedisMinIdleConns     int      `toml:"redis_min_idle_conns"`

	// DB Connection parameters
	DbMaxOpenConns       int `toml:"db_max_open_conns"`
	DbMaxIdleConns       int `toml:"db_max_idle_conns"`
	DbConnMaxLifeSeconds int `toml:"db_conn_max_life_seconds"`

	// If the value is not 0, the cached ping detection will be turned on, and the interval is the number of seconds.
	CacheCircuitCheckInterval int `toml:"cache_circuit_check_interval"`
	// This property sets the amount of seconds, after tripping the circuit,
	// to reject requests before allowing attempts again to determine if the circuit should again be closed.
	CacheCircuitCloseSleepWindow int `toml:"cache_circuit_close_sleep_window"`
	// This value is how may consecutive passing requests are required before the circuit is closed
	CacheCircuitCloseRequiredCount int `toml:"cache_circuit_close_required_count"`
	// This property sets the minimum number of requests in a rolling window that will trip the circuit.
	CacheCircuitOpenThreshold     int   `toml:"cache_circuit_open_threshold"`
	CacheCircuitExecTimeout       uint  `toml:"cache_circuit_exec_timeout"`
	CacheCircuitExecMaxConcurrent int64 `toml:"cache_circuit_exec_max_concurrent"`

	StsEncryptionKey string `toml:"sts_encryption_key"`

	DownloadBufPoolSize int64 `toml:"download_buf_pool_size"`
	UploadMinChunkSize  int64 `toml:"upload_min_chunk_size"`
	UploadMaxChunkSize  int64 `toml:"upload_max_chunk_size"`
	BigFileThreshold    int64 `toml:"big_file_threshold"`

	EnableQoS            bool `toml:"enable_qos"`
	DefaultReadOps       int  `toml:"default_read_ops"`
	DefaultWriteOps      int  `toml:"default_write_ops"`
	DefaultBandwidthKBps int  `toml:"default_bandwidth_kbps"`

	// The switch that decides whether to use the real thawing logic,
	// if it is on, the thawing logic is a false thawing mode that only modifies the database state
	RestoreMigratesFile bool     `toml:"restore_migrates_file"`
	LockTime            int      `toml:"lock_time"`
	RefreshLockTime     int      `toml:"refresh_lock_time"`
	LogDeliveryGroup    []string `toml:"log_delivery_group"`
}

var CONFIG Config

type PluginConfig

type PluginConfig struct {
	Path   string                 `toml:"path"`
	Enable bool                   `toml:"enable"`
	Args   map[string]interface{} `toml:"args"`
}