types

package
v1.1.70 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Dec 20, 2024 License: Apache-2.0 Imports: 20 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

View Source 
var (
	Cipher = env.GetString("DATASOURCE_CIPHER", "6b6579736f6d6574616c6b6579736f6d")
)

Functions

func ColumnExists 

func ColumnExists(db *gorm.DB, tableName string, columnName string) (bool, error)

func InitSQLDB 

func InitSQLDB(config *DB, gormDB *gorm.DB)

InitSQLDB initialize local sql by open sql and create task_infos table

func MigrateAlterBID 

func MigrateAlterBID(db *gorm.DB, tables iter.Seq[string]) error

MigrateAlterBID Add the `b_id“ field to certain database tables for versions prior to v3.7 (inclusive). We need to perform this operation manually, otherwise the `db.AutoMigrate` method will throw an exception

func OpenDB 

func OpenDB(opts *DB) (*gorm.DB, error)

func ParseDSN 

func ParseDSN(opts *DB) (dsn string, err error)

Types

type Client 

type Client struct {
	Version                  string  `json:"version,omitempty" validate:"required" yaml:"version"`
	Address                  string  `json:"address,omitempty" validate:"required" yaml:"address"`
	User                     string  `json:"user,omitempty" validate:"required" yaml:"user"`
	Password                 string  `json:"password,omitempty" validate:"required" yaml:"password"`
	ConcurrencyPerAddress    int     `json:"concurrencyPerAddress,optional" yaml:"concurrencyPerAddress,omitempty"`
	ReconnectInitialInterval *string `json:"reconnectInitialInterval,optional,omitempty" yaml:"reconnectInitialInterval,omitempty"`
	Retry                    int     `json:"retry,optional" yaml:"retry,omitempty"`
	RetryInitialInterval     *string `json:"retryInitialInterval,optional,omitempty" yaml:"retryInitialInterval,omitempty"`
}

type CreateImportTaskData 

type CreateImportTaskData struct {
	Id string `json:"id"`
}

type CreateImportTaskReq 

type CreateImportTaskReq struct {
	Id     *string             `json:"id,optional,omitempty" yaml:"id,omitempty"`
	Name   string              `json:"name" validate:"required" yaml:"name"`
	Tag    string              `json:"tag,omitempty,optional" yaml:"tag,omitempty"`
	Config *ImportTaskV2Config `json:"config" validate:"required" yaml:"config"`
}

type DB 

type DB struct {
	LogLevel                  int                    `json:"logLevel,default=1"`
	IgnoreRecordNotFoundError bool                   `json:"ignoreRecordNotFoundError,default=true"`
	AutoMigrate               bool                   `json:"autoMigrate,default=true"`
	Type                      string                 `json:"type,default=sqlite3"`
	Host                      string                 `json:"host,optional"`
	Name                      string                 `json:"name,optional"`
	User                      string                 `json:"user,optional"`
	Password                  string                 `json:"password,optional"`
	SqliteDbFilePath          string                 `json:"sqliteDbFilePath,default=./data/tasks.db"`
	MaxOpenConns              int                    `json:"maxOpenConns,default=30"`
	MaxIdleConns              int                    `json:"maxIdleConns,default=10"`
	Migrates                  map[string]interface{} `json:"-"`
}

type DeleteImportTaskReq 

type DeleteImportTaskReq struct {
	Id string `path:"id"`
}

type GetImportTaskData 

type GetImportTaskData struct {
	Id            string          `json:"id"`
	Name          string          `json:"name"`
	User          string          `json:"user"`
	Address       string          `json:"address"`
	ImportAddress []string        `json:"importAddress"`
	Space         string          `json:"space"`
	Status        string          `json:"status"`
	Message       string          `json:"message"`
	CreateTime    int64           `json:"createTime"`
	UpdateTime    int64           `json:"updateTime"`
	Stats         ImportTaskStats `json:"stats"`
	RawConfig     string          `json:"rawConfig"`
}

type GetImportTaskReq 

type GetImportTaskReq struct {
	Id string `path:"id" validate:"required"`
}

type GetManyImportTaskData 

type GetManyImportTaskData struct {
	Total int64               `json:"total"`
	List  []GetImportTaskData `json:"list"`
}

type GetManyImportTaskLogReq 

type GetManyImportTaskLogReq struct {
	Id string `path:"id" validate:"required"`
}

type GetManyImportTaskReq 

type GetManyImportTaskReq struct {
	Page     int    `form:"page,default=1"`
	PageSize int    `form:"pageSize,default=20"`
	Space    string `form:"space,optional"`
	Status   string `form:"status,optional"`
}

type ImportTaskCSV 

type ImportTaskCSV struct {
	WithHeader *bool   `json:"withHeader,optional"`
	LazyQuotes *bool   `json:"lazyQuotes,optional"`
	Delimiter  *string `json:"delimiter,optional"`
}

type ImportTaskStats 

type ImportTaskStats struct {
	Processed       int64 `json:"processed"`
	Total           int64 `json:"total"`
	FailedRecords   int64 `json:"failedRecords"`
	TotalRecords    int64 `json:"totalRecords"`
	FailedRequest   int64 `json:"failedRequest"`
	TotalRequest    int64 `json:"totalRequest"`
	TotalLatency    int64 `json:"totalLatency"`
	TotalRespTime   int64 `json:"totalRespTime"`
	FailedProcessed int64 `json:"failedProcessed"`
	TotalProcessed  int64 `json:"totalProcessed"`
}

type ImportTaskV2Config 

type ImportTaskV2Config struct {
	configv3.Config `yaml:",inline" json:",inline"`
	Cron            string `json:"cron,omitempty,optional" yaml:"cron,omitempty"`
}

type Log 

type Log struct {
	Level   *string    `json:"level,omitempty,optional,default=error" yaml:"level,omitempty"`
	Console *bool      `json:"console,omitempty,optional,default=true" yaml:"console,omitempty"`
	Files   []string   `json:"files,omitempty,optional" yaml:"files,omitempty"`
	Fields  []LogField `json:"fields,omitempty,optional" yaml:"fields,omitempty"`
}

type LogField 

type LogField struct {
	Key   string      `json:"key" yaml:"key,omitempty"`
	Value interface{} `json:"value" yaml:"value,omitempty"`
}

type Manager 

type Manager struct {
	SpaceName           string  `json:"spaceName,omitempty" validate:"required" yaml:"spaceName"`
	Batch               int     `json:"batch,omitempty,optional,default=200" yaml:"batch,omitempty"`
	ReaderConcurrency   int     `json:"readerConcurrency,omitempty,optional,default=10" yaml:"readerConcurrency,omitempty"`
	ImporterConcurrency int     `json:"importerConcurrency,omitempty,optional,default=10" yaml:"importerConcurrency,omitempty"`
	StatsInterval       *string `json:"statsInterval,omitempty,optional,default=10s" yaml:"statsInterval,omitempty"`
}

type Nebula 

type Nebula struct {
	Address         string `json:"address" validate:"required"`
	User            string `json:"user" validate:"required"`
	Password        string `json:"password" validate:"required"`
	MaxConnPoolSize int    `json:"maxConnPoolSize,omitempty,optional,default=100"`
	MinConnPoolSize int    `json:"minConnPoolSize,omitempty,optional,default=10"`
	IdleTimeSeconds int    `json:"idleTimeSeconds,omitempty,optional,default=28800"`
	TimeOutSeconds  int    `json:"timeOutSeconds,omitempty,optional,default=10"`
	RiskNGQLRegexp  string `json:"riskNGQLRegexp,omitempty,optional"`
}

type Redis 

type Redis struct {
	// The network type, either tcp or unix.
	// Default is tcp.
	Network string `json:"network,default=tcp"`
	URL     string `json:"url,default=redis://:@localhost:6379/0"`

	// ClientName will execute the `CLIENT SETNAME ClientName` command for each conn.
	ClientName string `json:"clientName,default=nebula-studio-gac"`

	// Protocol 2 or 3. Use the version to negotiate RESP version with redis-server.
	// Default is 3.
	Protocol int `json:"protocol,default=3"`

	// Maximum number of retries before giving up.
	// Default is 3 retries; -1 (not 0) disables retries.
	MaxRetries int `json:"maxRetries,default=3"`

	// Base number of socket connections.
	// Default is 10 connections per every available CPU as reported by runtime.GOMAXPROCS.
	// If there is not enough connections in the pool, new connections will be allocated in excess of PoolSize,
	// you can limit it through MaxActiveConns
	PoolSize int `json:"poolSize,default=100"`
	// Minimum number of idle connections which is useful when establishing
	// new connection is slow.
	// Default is 0. the idle connections are not closed by default.
	MinIdleConns int `json:"minIdleConns,default=0"`
	// Maximum number of idle connections.
	// Default is 0. the idle connections are not closed by default.
	MaxIdleConns int `json:"maxIdleConns,default=0"`
}

type RestartImportTaskReq 

type RestartImportTaskReq struct {
	Id *string `path:"id" validate:"required"`
}

type StopImportTaskReq 

type StopImportTaskReq struct {
	Id string `path:"id"`
}

type TaskConfig 

type TaskConfig struct {
	Dir    *TaskDir `json:"dir,optional"`
	Nebula *Nebula  `json:"nebula" validate:"required"`
	DB     *DB      `json:"db" validate:"required"`
	Redis  *Redis   `json:"redis" validate:"required"`
	// contains filtered or unexported fields
}

func (*TaskConfig) CreateDb 

func (tc *TaskConfig) CreateDb() (gormDB *gorm.DB, err error)

func (*TaskConfig) CreateRedis 

func (tc *TaskConfig) CreateRedis() *redis.Client

type TaskDir 

type TaskDir struct {
	UploadDir      string `json:"uploadDir,default=./data/upload/"`
	TasksDir       string `json:"tasksDir,default=./data/tasks"`
	CompressionDIR string `json:"compressionDIR,default=./data/compression"`
}

type TaskStatus 

type TaskStatus int
const (
	StatusUnknown TaskStatus = iota
	Finished
	Stopped
	Scheduling
	Running
	NotExisted
	Aborted
	Draft
)

the task in memory (map) has 2 status: processing, aborted; and the task in local sql has 2 status: finished, stoped;

func NewTaskStatus 

func NewTaskStatus(status string) TaskStatus

func (TaskStatus) String 

func (status TaskStatus) String() string

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.