utils

package

v0.6.2 Latest Latest Go to latest Published: Jun 12, 2023 License: Apache-2.0 Imports: 56 Imported by: 1

Details

Valid go.mod file
Redistributable license
Tagged version
Stable version
Learn more about best practices

Repository

github.com/crawlab-team/crawlab-core

Links

Open Source Insights

Documentation ¶

Index ¶

Variables
func BackoffErrorNotify(prefix string) backoff.Notify
func BsonMEqual(v1, v2 bson.M) (ok bool)
func BytesToString(b []byte) string
func CleanupDemo() (err error)
func Close(c io.Closer)
func Compress(files []*os.File, dest string) error
func ComputeHmacSha256(message string, secret string) string
func Contains(array interface{}, val interface{}) (fla bool)
func CopyDir(src string, dst string) error
func CopyFile(src, dst string) error
func CreateDirPath(filePath string)
func DeCompress(srcFile *os.File, dstPath string) error
func DeCompressByPath(tarFile, dest string) error
func DecryptAES(src string) (res string, err error)
func DefaultWait()
func DenormalizeBsonMObjectId(m bson.M) (res bson.M)
func EncryptAES(src string) (res string, err error)
func EncryptMd5(str string) string
func EnvIsTrue(key string, defaultOk bool) bool
func Exists(path string) bool
func FillEmptyFileData(data []byte) (res []byte)
func FilterToQuery(f interfaces.Filter) (q bson.M, err error)
func ForceInitModule(fn func() error) (err error)
func GetAllFilesFromDir(dirPath string) ([]*os.File, error)
func GetApiAddress() (res string)
func GetArrayItems(array interface{}) (res []interface{}, err error)
func GetCockroachdbSession(ds *models.DataSource) (s db.Session, err error)
func GetCockroachdbSessionWithTimeout(ds *models.DataSource, timeout time.Duration) (s db.Session, err error)
func GetElasticsearchClient(ds *models.DataSource) (c *elasticsearch.Client, err error)
func GetElasticsearchClientWithTimeout(ds *models.DataSource, timeout time.Duration) (c *elasticsearch.Client, err error)
func GetElasticsearchQuery(query generic.ListQuery) (buf *bytes.Buffer)
func GetElasticsearchQueryWithOptions(query generic.ListQuery, opts *generic.ListOptions) (buf *bytes.Buffer)
func GetFilesFromDir(dirPath string) ([]*os.File, error)
func GetFromDbCache(key string, getFn func() (string, error)) (res string, err error)
func GetKafkaConnection(ds *models.DataSource) (c *kafka.Conn, err error)
func GetKafkaConnectionWithTimeout(ds *models.DataSource, timeout time.Duration) (c *kafka.Conn, err error)
func GetLocalTime(t time.Time) time.Time
func GetLocalTimeString(t time.Time) string
func GetMongoClient(ds *models.DataSource) (c *mongo2.Client, err error)
func GetMongoClientWithTimeout(ds *models.DataSource, timeout time.Duration) (c *mongo2.Client, err error)
func GetMongoOpts(opts *generic.ListOptions) (res *mongo.FindOptions)
func GetMongoQuery(query generic.ListQuery) (res bson.M)
func GetMssqlSession(ds *models.DataSource) (s db.Session, err error)
func GetMssqlSessionWithTimeout(ds *models.DataSource, timeout time.Duration) (s db.Session, err error)
func GetMysqlSession(ds *models.DataSource) (s db.Session, err error)
func GetMysqlSessionWithTimeout(ds *models.DataSource, timeout time.Duration) (s db.Session, err error)
func GetNodeType() string
func GetPostgresqlSession(ds *models.DataSource) (s db.Session, err error)
func GetPostgresqlSessionWithTimeout(ds *models.DataSource, timeout time.Duration) (s db.Session, err error)
func GetResultHash(value interface{}, keys []string) (res string, err error)
func GetRpcParam(key string, params map[string]string) string
func GetSecretKey() string
func GetSecretKeyBytes() []byte
func GetSpiderCol(col string, name string) string
func GetSpiderMd5Str(file string) string
func GetSqlQuery(query generic.ListQuery) (res db.Cond)
func GetSqliteSession(ds *models.DataSource) (s db.Session, err error)
func GetSqliteSessionWithTimeout(ds *models.DataSource, timeout time.Duration) (s db.Session, err error)
func GetTimeString(t time.Time) string
func GetUserFromArgs(args ...interface{}) (u interfaces.User)
func ImportDemo() (err error)
func InitGitClientAuth(g *models.Git, gitClient *vcs.GitClient)
func InitModule(id interfaces.ModuleId, fn func() error) (err error)
func InitializedDemo() (ok bool)
func IsCancellable(status string) bool
func IsDebug() bool
func IsDemo() (ok bool)
func IsDir(path string) bool
func IsDocker() (ok bool)
func IsFile(path string) bool
func IsMaster() bool
func JsonToBytes(d interface{}) (bytes []byte, err error)
func ListDir(path string) []os.FileInfo
func LogDebug(msg string)
func NewUUIDString() (res string)
func NormalizeBsonMObjectId(m bson.M) (res bson.M)
func NormalizeObjectId(v interface{}) (res interface{})
func ObjectToString(params interface{}) string
func OpenFile(fileName string) *os.File
func ReadFileOneLine(fileName string) string
func ReimportDemo() (err error)
func RemoveFiles(path string)
func SetFileVariable(filePath string, key string, value string) error
func ShuffleArray(slice []interface{}) (err error)
func StringArrayContains(arr []string, str string) bool
func TrimFileData(data []byte) (res []byte)
func VisualizeContainer(c *dig.Container) (err error)
type ChanMap
- func NewChanMap() *ChanMap
- func (cm *ChanMap) Chan(key string) chan string
- func (cm *ChanMap) ChanBlocked(key string) chan string
- func (cm *ChanMap) HasChanKey(key string) bool

Constants ¶

This section is empty.

Variables ¶

View Source

var CronUtils = cronUtils{
	// contains filtered or unexported fields
}

View Source

var TaskExecChanMap = NewChanMap()

Functions ¶

func BackoffErrorNotify ¶ added in v0.6.0

func BackoffErrorNotify(prefix string) backoff.Notify

func BsonMEqual ¶ added in v0.6.0

func BsonMEqual(v1, v2 bson.M) (ok bool)

func BytesToString ¶

func BytesToString(b []byte) string

func CleanupDemo ¶ added in v0.6.1

func CleanupDemo() (err error)

func Close ¶

func Close(c io.Closer)

func Compress ¶

func Compress(files []*os.File, dest string) error

压缩文件 files 文件数组，可以是不同dir下的文件或者文件夹 dest 压缩文件存放地址

func ComputeHmacSha256 ¶

func ComputeHmacSha256(message string, secret string) string

func Contains ¶

func Contains(array interface{}, val interface{}) (fla bool)

func CopyDir ¶

func CopyDir(src string, dst string) error

Dir copies a whole directory recursively

func CopyFile ¶

func CopyFile(src, dst string) error

File copies a single file from src to dst

func CreateDirPath ¶

func CreateDirPath(filePath string)

创建文件夹

func DeCompress ¶

func DeCompress(srcFile *os.File, dstPath string) error

func DeCompressByPath ¶

func DeCompressByPath(tarFile, dest string) error

func DecryptAES ¶ added in v0.6.0

func DecryptAES(src string) (res string, err error)

func DefaultWait ¶ added in v0.6.0

func DefaultWait()

func DenormalizeBsonMObjectId ¶ added in v0.6.0

func DenormalizeBsonMObjectId(m bson.M) (res bson.M)

func EncryptAES ¶ added in v0.6.0

func EncryptAES(src string) (res string, err error)

func EncryptMd5 ¶ added in v0.6.0

func EncryptMd5(str string) string

func EnvIsTrue ¶ added in v0.6.1

func EnvIsTrue(key string, defaultOk bool) bool

func Exists ¶

func Exists(path string) bool

func FillEmptyFileData ¶ added in v0.6.0

func FillEmptyFileData(data []byte) (res []byte)

func FilterToQuery ¶ added in v0.6.1

func FilterToQuery(f interfaces.Filter) (q bson.M, err error)

FilterToQuery Translate entity.Filter to bson.M

func ForceInitModule ¶ added in v0.6.0

func ForceInitModule(fn func() error) (err error)

func GetAllFilesFromDir ¶

func GetAllFilesFromDir(dirPath string) ([]*os.File, error)

func GetApiAddress ¶ added in v0.6.1

func GetApiAddress() (res string)

func GetArrayItems ¶ added in v0.6.0

func GetArrayItems(array interface{}) (res []interface{}, err error)

func GetCockroachdbSession ¶ added in v0.6.2

func GetCockroachdbSession(ds *models.DataSource) (s db.Session, err error)

func GetCockroachdbSessionWithTimeout ¶ added in v0.6.2

func GetCockroachdbSessionWithTimeout(ds *models.DataSource, timeout time.Duration) (s db.Session, err error)

func GetElasticsearchClient ¶ added in v0.6.2

func GetElasticsearchClient(ds *models.DataSource) (c *elasticsearch.Client, err error)

func GetElasticsearchClientWithTimeout ¶ added in v0.6.2

func GetElasticsearchClientWithTimeout(ds *models.DataSource, timeout time.Duration) (c *elasticsearch.Client, err error)

func GetElasticsearchQuery ¶ added in v0.6.2

func GetElasticsearchQuery(query generic.ListQuery) (buf *bytes.Buffer)

func GetElasticsearchQueryWithOptions ¶ added in v0.6.2

func GetElasticsearchQueryWithOptions(query generic.ListQuery, opts *generic.ListOptions) (buf *bytes.Buffer)

func GetFilesFromDir ¶

func GetFilesFromDir(dirPath string) ([]*os.File, error)

func GetFromDbCache ¶ added in v0.6.0

func GetFromDbCache(key string, getFn func() (string, error)) (res string, err error)

func GetKafkaConnection ¶ added in v0.6.2

func GetKafkaConnection(ds *models.DataSource) (c *kafka.Conn, err error)

func GetKafkaConnectionWithTimeout ¶ added in v0.6.2

func GetKafkaConnectionWithTimeout(ds *models.DataSource, timeout time.Duration) (c *kafka.Conn, err error)

func GetLocalTime ¶

func GetLocalTime(t time.Time) time.Time

func GetLocalTimeString ¶

func GetLocalTimeString(t time.Time) string

func GetMongoClient ¶ added in v0.6.2

func GetMongoClient(ds *models.DataSource) (c *mongo2.Client, err error)

func GetMongoClientWithTimeout ¶ added in v0.6.2

func GetMongoClientWithTimeout(ds *models.DataSource, timeout time.Duration) (c *mongo2.Client, err error)

func GetMongoOpts ¶ added in v0.6.0

func GetMongoOpts(opts *generic.ListOptions) (res *mongo.FindOptions)

func GetMongoQuery ¶ added in v0.6.0

func GetMongoQuery(query generic.ListQuery) (res bson.M)

func GetMssqlSession ¶ added in v0.6.2

func GetMssqlSession(ds *models.DataSource) (s db.Session, err error)

func GetMssqlSessionWithTimeout ¶ added in v0.6.2

func GetMssqlSessionWithTimeout(ds *models.DataSource, timeout time.Duration) (s db.Session, err error)

func GetMysqlSession ¶ added in v0.6.2

func GetMysqlSession(ds *models.DataSource) (s db.Session, err error)

func GetMysqlSessionWithTimeout ¶ added in v0.6.2

func GetMysqlSessionWithTimeout(ds *models.DataSource, timeout time.Duration) (s db.Session, err error)

func GetNodeType ¶ added in v0.6.1

func GetNodeType() string

func GetPostgresqlSession ¶ added in v0.6.2

func GetPostgresqlSession(ds *models.DataSource) (s db.Session, err error)

func GetPostgresqlSessionWithTimeout ¶ added in v0.6.2

func GetPostgresqlSessionWithTimeout(ds *models.DataSource, timeout time.Duration) (s db.Session, err error)

func GetResultHash ¶ added in v0.6.1

func GetResultHash(value interface{}, keys []string) (res string, err error)

func GetRpcParam ¶

func GetRpcParam(key string, params map[string]string) string

获取 RPC 参数

func GetSecretKey ¶ added in v0.6.0

func GetSecretKey() string

func GetSecretKeyBytes ¶ added in v0.6.0

func GetSecretKeyBytes() []byte

func GetSpiderCol ¶

func GetSpiderCol(col string, name string) string

func GetSpiderMd5Str ¶

func GetSpiderMd5Str(file string) string

func GetSqlQuery ¶ added in v0.6.2

func GetSqlQuery(query generic.ListQuery) (res db.Cond)

func GetSqliteSession ¶ added in v0.6.2

func GetSqliteSession(ds *models.DataSource) (s db.Session, err error)

func GetSqliteSessionWithTimeout ¶ added in v0.6.2

func GetSqliteSessionWithTimeout(ds *models.DataSource, timeout time.Duration) (s db.Session, err error)

func GetTimeString ¶

func GetTimeString(t time.Time) string

func GetUserFromArgs ¶ added in v0.6.0

func GetUserFromArgs(args ...interface{}) (u interfaces.User)

func ImportDemo ¶ added in v0.6.1

func ImportDemo() (err error)

func InitGitClientAuth ¶ added in v0.6.1

func InitGitClientAuth(g *models.Git, gitClient *vcs.GitClient)

func InitModule ¶ added in v0.6.0

func InitModule(id interfaces.ModuleId, fn func() error) (err error)

func InitializedDemo ¶ added in v0.6.1

func InitializedDemo() (ok bool)

func IsCancellable ¶ added in v0.6.0

func IsCancellable(status string) bool

func IsDebug ¶ added in v0.6.0

func IsDebug() bool

func IsDemo ¶ added in v0.6.1

func IsDemo() (ok bool)

func IsDir ¶

func IsDir(path string) bool

func IsDocker ¶ added in v0.6.1

func IsDocker() (ok bool)

func IsFile ¶

func IsFile(path string) bool

func IsMaster ¶ added in v0.6.1

func IsMaster() bool

func JsonToBytes ¶ added in v0.6.0

func JsonToBytes(d interface{}) (bytes []byte, err error)

func ListDir ¶

func ListDir(path string) []os.FileInfo

func LogDebug ¶ added in v0.6.0

func LogDebug(msg string)

func NewUUIDString ¶ added in v0.6.0

func NewUUIDString() (res string)

func NormalizeBsonMObjectId ¶ added in v0.6.0

func NormalizeBsonMObjectId(m bson.M) (res bson.M)

func NormalizeObjectId ¶ added in v0.6.1

func NormalizeObjectId(v interface{}) (res interface{})

func ObjectToString ¶

func ObjectToString(params interface{}) string

Object 转化为 String

func OpenFile ¶

func OpenFile(fileName string) *os.File

func ReadFileOneLine ¶

func ReadFileOneLine(fileName string) string

func ReimportDemo ¶ added in v0.6.1

func ReimportDemo() (err error)

func RemoveFiles ¶

func RemoveFiles(path string)

func SetFileVariable ¶

func SetFileVariable(filePath string, key string, value string) error

设置文件变量值可以理解为将文件中的变量占位符替换为想要设置的值

func ShuffleArray ¶ added in v0.6.0

func ShuffleArray(slice []interface{}) (err error)

func StringArrayContains ¶

func StringArrayContains(arr []string, str string) bool

func TrimFileData ¶ added in v0.6.0

func TrimFileData(data []byte) (res []byte)

func VisualizeContainer ¶ added in v0.6.0

func VisualizeContainer(c *dig.Container) (err error)

Types ¶

type ChanMap ¶

type ChanMap struct {
	// contains filtered or unexported fields
}

func NewChanMap ¶

func NewChanMap() *ChanMap

func (*ChanMap) Chan ¶

func (cm *ChanMap) Chan(key string) chan string

func (*ChanMap) ChanBlocked ¶

func (cm *ChanMap) ChanBlocked(key string) chan string

func (*ChanMap) HasChanKey ¶

func (cm *ChanMap) HasChanKey(key string) bool

Source Files ¶

View all Source files

Directories ¶

Path	Synopsis
binders

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL