Documentation ¶
Index ¶
- Constants
- type Address
- type Color
- type ColorService
- type Entity
- type FsFileInfo
- type FsService
- type GrpcBase
- type GrpcBaseServiceParams
- type GrpcClient
- type GrpcClientModelBaseService
- type GrpcClientModelDelegate
- type GrpcClientModelNodeService
- type GrpcClientModelService
- type GrpcClientModelSpiderService
- type GrpcClientModelTaskService
- type GrpcClientModelTaskStatService
- type GrpcClientPool
- type GrpcModelBaseServiceMessage
- type GrpcModelBinder
- type GrpcModelDelegateMessage
- type GrpcModelListBinder
- type GrpcServer
- type GrpcSubscribe
- type Injectable
- type List
- type Model
- type ModelArtifact
- type ModelArtifactSys
- type ModelBaseService
- type ModelBinder
- type ModelDelegate
- type ModelDelegateMethod
- type ModelId
- type ModelListBinder
- type ModelNodeDelegate
- type ModelService
- type ModelWithTags
- type Module
- type ModuleId
- type Node
- type NodeConfigService
- type NodeMasterService
- type NodeService
- type NodeServiceOption
- type NodeWorkerService
- type PluginService
- type Provide
- type Result
- type ResultService
- type Schedule
- type ScheduleService
- type ServiceCrudOption
- type ServiceCrudOptions
- type Spider
- type SpiderAdminService
- type SpiderCloneOptions
- type SpiderFsService
- type SpiderRunOptions
- type SpiderSyncService
- type StatsService
- type Tag
- type Task
- type TaskBaseService
- type TaskHandlerService
- type TaskRunner
- type TaskSchedulerService
- type TaskStat
- type TaskStatsService
- type Test
- type User
- type UserCreateOptions
- type UserLoginOptions
- type UserService
- type WithAddress
- type WithConfigPath
- type WithModelId
Constants ¶
View Source
const ( ModelIdArtifact = iota ModelIdTag ModelIdNode ModelIdProject ModelIdSpider ModelIdTask ModelIdJob ModelIdSchedule ModelIdUser ModelIdSetting ModelIdToken ModelIdVariable ModelIdTaskQueue ModelIdTaskStat ModelIdPlugin ModelIdSpiderStat ModelIdDataSource ModelIdDataCollection ModelIdResult ModelIdPassword )
View Source
const ( ModelColNameArtifact = "artifacts" ModelColNameTag = "tags" ModelColNameNode = "nodes" ModelColNameProject = "projects" ModelColNameSpider = "spiders" ModelColNameTask = "tasks" ModelColNameJob = "jobs" ModelColNameSchedule = "schedules" ModelColNameUser = "users" ModelColNameSetting = "settings" ModelColNameToken = "tokens" ModelColNameVariable = "variables" ModelColNameTaskQueue = "task_queue" ModelColNameTaskStat = "task_stats" ModelColNamePlugin = "plugins" ModelColNameSpiderStat = "spider_stats" ModelColNameDataSource = "data_sources" ModelColNameDataCollection = "data_collections" ModelColNamePasswords = "passwords" )
View Source
const ( ModelDelegateMethodAdd = iota ModelDelegateMethodSave ModelDelegateMethodDelete ModelDelegateMethodGetArtifact ModelDelegateMethodRefresh )
Variables ¶
This section is empty.
Functions ¶
This section is empty.
Types ¶
type ColorService ¶
type FsFileInfo ¶
type FsService ¶
type FsService interface { WithConfigPath List(path string, opts ...ServiceCrudOption) (files []FsFileInfo, err error) GetFile(path string, opts ...ServiceCrudOption) (data []byte, err error) GetFileInfo(path string, opts ...ServiceCrudOption) (file FsFileInfo, err error) Save(path string, data []byte, opts ...ServiceCrudOption) (err error) Rename(path, newPath string, opts ...ServiceCrudOption) (err error) Delete(path string, opts ...ServiceCrudOption) (err error) Copy(path, newPath string, opts ...ServiceCrudOption) (err error) Commit(msg string) (err error) SyncToFs(opts ...ServiceCrudOption) (err error) SyncToWorkspace() (err error) GetFsPath() (path string) SetFsPath(path string) GetWorkspacePath() (path string) SetWorkspacePath(path string) GetRepoPath() (path string) SetRepoPath(path string) GetFs() (fs cfs.Manager) }
type GrpcBase ¶
type GrpcBase interface { WithConfigPath Init() (err error) Start() (err error) Stop() (err error) Register() (err error) }
type GrpcBaseServiceParams ¶
type GrpcBaseServiceParams interface { Entity }
type GrpcClient ¶
type GrpcClient interface { GrpcBase WithConfigPath GetModelDelegateClient() grpc.ModelDelegateClient GetModelBaseServiceClient() grpc.ModelBaseServiceClient GetNodeClient() grpc.NodeServiceClient GetTaskClient() grpc.TaskServiceClient GetPluginClient() grpc.PluginServiceClient SetAddress(Address) SetTimeout(time.Duration) Context() (context.Context, context.CancelFunc) NewRequest(interface{}) *grpc.Request GetMessageChannel() chan *grpc.StreamMessage Restart() error NewModelBaseServiceRequest(ModelId, GrpcBaseServiceParams) (*grpc.Request, error) IsStarted() bool IsClosed() bool Err() error }
type GrpcClientModelBaseService ¶
type GrpcClientModelBaseService interface { WithModelId WithConfigPath ModelBaseService }
type GrpcClientModelDelegate ¶
type GrpcClientModelDelegate interface { ModelDelegate WithConfigPath Close() error }
type GrpcClientModelNodeService ¶
type GrpcClientModelNodeService interface { ModelBaseService GetNodeById(id primitive.ObjectID) (n Node, err error) GetNode(query bson.M, opts *mongo.FindOptions) (n Node, err error) GetNodeByKey(key string) (n Node, err error) GetNodeList(query bson.M, opts *mongo.FindOptions) (res []Node, err error) }
type GrpcClientModelService ¶
type GrpcClientModelService interface { WithConfigPath NewBaseServiceDelegate(id ModelId) (GrpcClientModelBaseService, error) }
type GrpcClientPool ¶
type GrpcClientPool interface { WithConfigPath Init() error NewClient() error GetClient() (GrpcClient, error) SetSize(int) }
type GrpcModelBinder ¶
type GrpcModelBinder interface {
// contains filtered or unexported methods
}
type GrpcModelDelegateMessage ¶
type GrpcModelDelegateMessage interface { GetModelId() ModelId GetMethod() ModelDelegateMethod GetData() []byte ToBytes() (data []byte) }
type GrpcModelListBinder ¶
type GrpcModelListBinder interface {
// contains filtered or unexported methods
}
type GrpcServer ¶
type GrpcServer interface { GrpcBase SetAddress(Address) GetSubscribe(key string) (sub GrpcSubscribe, err error) SetSubscribe(key string, sub GrpcSubscribe) DeleteSubscribe(key string) SendStreamMessage(nodeKey string, code grpc.StreamMessageCode) (err error) SendStreamMessageWithData(nodeKey string, code grpc.StreamMessageCode, d interface{}) (err error) IsStopped() (res bool) }
type GrpcSubscribe ¶
type GrpcSubscribe interface { GetStream() grpc.NodeService_SubscribeServer GetFinished() chan bool }
type Injectable ¶
type Injectable interface {
Inject() error
}
type ModelArtifact ¶
type ModelArtifactSys ¶
type ModelArtifactSys interface { GetCreateTs() time.Time SetCreateTs(ts time.Time) GetUpdateTs() time.Time SetUpdateTs(ts time.Time) GetDeleteTs() time.Time SetDeleteTs(ts time.Time) GetCreateUid() primitive.ObjectID SetCreateUid(id primitive.ObjectID) GetUpdateUid() primitive.ObjectID SetUpdateUid(id primitive.ObjectID) GetDeleteUid() primitive.ObjectID SetDeleteUid(id primitive.ObjectID) }
type ModelBaseService ¶
type ModelBaseService interface { GetModelId() (id ModelId) SetModelId(id ModelId) GetById(id primitive.ObjectID) (res Model, err error) Get(query bson.M, opts *mongo.FindOptions) (res Model, err error) GetList(query bson.M, opts *mongo.FindOptions) (res arraylist.List, err error) DeleteById(id primitive.ObjectID) (err error) Delete(query bson.M) (err error) DeleteList(query bson.M) (err error) ForceDeleteList(query bson.M) (err error) UpdateById(id primitive.ObjectID, update bson.M) (err error) Update(query bson.M, update bson.M, fields []string) (err error) UpdateDoc(query bson.M, doc Model, fields []string) (err error) Insert(docs ...interface{}) (err error) Count(query bson.M) (total int, err error) }
type ModelBinder ¶
type ModelDelegate ¶
type ModelDelegateMethod ¶
type ModelDelegateMethod int
type ModelListBinder ¶
type ModelNodeDelegate ¶
type ModelService ¶
type ModelService interface {
GetBaseService(id ModelId) (svc ModelBaseService)
}
type ModelWithTags ¶
type Node ¶
type Node interface { ModelWithTags GetKey() (key string) GetIsMaster() (ok bool) SetActive(active bool) SetActiveTs(activeTs time.Time) GetStatus() (status string) SetStatus(status string) GetEnabled() (enabled bool) SetEnabled(enabled bool) GetAvailableRunners() (runners int) SetAvailableRunners(runners int) GetMaxRunners() (runners int) SetMaxRunners(runners int) IncrementAvailableRunners() DecrementAvailableRunners() }
type NodeConfigService ¶
type NodeMasterService ¶
type NodeMasterService interface { NodeService Monitor() SetMonitorInterval(duration time.Duration) Register() error StopOnError() GetServer() GrpcServer }
type NodeService ¶
type NodeService interface { Module WithConfigPath WithAddress GetConfigService() NodeConfigService }
type NodeServiceOption ¶
type NodeServiceOption interface { }
type NodeWorkerService ¶
type NodeWorkerService interface { NodeService Register() Recv() ReportStatus() SetHeartbeatInterval(duration time.Duration) }
type PluginService ¶
type ResultService ¶
type Schedule ¶
type Schedule interface { Model GetEnabled() (enabled bool) SetEnabled(enabled bool) GetEntryId() (id cron.EntryID) SetEntryId(id cron.EntryID) GetCron() (c string) SetCron(c string) GetSpiderId() (id primitive.ObjectID) SetSpiderId(id primitive.ObjectID) GetMode() (mode string) SetMode(mode string) GetNodeIds() (ids []primitive.ObjectID) SetNodeIds(ids []primitive.ObjectID) GetNodeTags() (tags []string) SetNodeTags(tags []string) GetCmd() (cmd string) SetCmd(cmd string) GetParam() (param string) SetParam(param string) GetPriority() (p int) SetPriority(p int) }
type ScheduleService ¶
type ScheduleService interface { WithConfigPath Module GetLocation() (loc *time.Location) SetLocation(loc *time.Location) GetDelay() (delay bool) SetDelay(delay bool) GetSkip() (skip bool) SetSkip(skip bool) GetUpdateInterval() (interval time.Duration) SetUpdateInterval(interval time.Duration) Enable(s Schedule) (err error) Disable(s Schedule) (err error) Update() GetCron() (c *cron.Cron) }
type ServiceCrudOption ¶
type ServiceCrudOption func(o *ServiceCrudOptions)
func WithNotSyncToWorkspace ¶
func WithNotSyncToWorkspace() ServiceCrudOption
func WithOnlyFromWorkspace ¶
func WithOnlyFromWorkspace() ServiceCrudOption
type ServiceCrudOptions ¶
type Spider ¶
type Spider interface { Model GetType() (ty string) GetMode() (mode string) SetMode(mode string) GetNodeIds() (ids []primitive.ObjectID) SetNodeIds(ids []primitive.ObjectID) GetNodeTags() (tags []string) SetNodeTags(tags []string) GetCmd() (cmd string) SetCmd(cmd string) GetParam() (param string) SetParam(param string) GetPriority() (p int) SetPriority(p int) }
type SpiderAdminService ¶
type SpiderAdminService interface { WithConfigPath // Schedule a new task of the spider Schedule(id primitive.ObjectID, opts *SpiderRunOptions) (err error) // Clone the spider Clone(id primitive.ObjectID, opts *SpiderCloneOptions) (err error) // Delete the spider Delete(id primitive.ObjectID) (err error) }
type SpiderCloneOptions ¶
type SpiderCloneOptions struct {
Name string
}
type SpiderFsService ¶
type SpiderFsService interface { WithConfigPath Init() (err error) SetId(id primitive.ObjectID) List(path string) (files []FsFileInfo, err error) GetFile(path string) (data []byte, err error) GetFileInfo(path string) (file FsFileInfo, err error) Save(path string, data []byte) (err error) Rename(path, newPath string) (err error) Delete(path string) (err error) Copy(path, newPath string) (err error) Commit(msg string) (err error) GetFsPath() (res string) GetWorkspacePath() (res string) GetRepoPath() (res string) SetFsPathBase(path string) SetWorkspacePathBase(path string) SetRepoPathBase(path string) GetFsService() (fsSvc FsService) }
type SpiderRunOptions ¶
type SpiderSyncService ¶
type SpiderSyncService interface { WithConfigPath SetFsPathBase(path string) SetWorkspacePathBase(path string) SetRepoPathBase(path string) GetFsService(id primitive.ObjectID) (fsSvc SpiderFsService, err error) ForceGetFsService(id primitive.ObjectID) (fsSvc SpiderFsService, err error) SyncToFs(id primitive.ObjectID) (err error) SyncToWorkspace(id primitive.ObjectID) (err error) }
type StatsService ¶
type Task ¶
type Task interface { Model GetNodeId() (id primitive.ObjectID) SetNodeId(id primitive.ObjectID) GetNodeIds() (ids []primitive.ObjectID) GetNodeTags() (tags []string) GetStatus() (status string) SetStatus(status string) GetError() (error string) SetError(error string) GetSpiderId() (id primitive.ObjectID) GetType() (ty string) GetCmd() (cmd string) GetParam() (param string) GetPriority() (p int) }
type TaskBaseService ¶
type TaskHandlerService ¶
type TaskHandlerService interface { TaskBaseService // Run task and execute locally Run(taskId primitive.ObjectID) (err error) // Cancel task locally Cancel(taskId primitive.ObjectID) (err error) // ReportStatus periodically report handler status to master ReportStatus() // Reset reset internals to default Reset() // IsSyncLocked whether the given spider is locked for files sync IsSyncLocked(spiderId primitive.ObjectID) (ok bool) // LockSync lock files sync for given spider LockSync(spiderId primitive.ObjectID) // UnlockSync unlock files sync for given spider UnlockSync(spiderId primitive.ObjectID) // GetExitWatchDuration get max runners GetExitWatchDuration() (duration time.Duration) // SetExitWatchDuration set max runners SetExitWatchDuration(duration time.Duration) // GetReportInterval get report interval GetReportInterval() (interval time.Duration) // SetReportInterval set report interval SetReportInterval(interval time.Duration) // GetModelService get model service GetModelService() (modelSvc GrpcClientModelService) // GetModelSpiderService get model spider service GetModelSpiderService() (modelSpiderSvc GrpcClientModelSpiderService) // GetModelTaskService get model task service GetModelTaskService() (modelTaskSvc GrpcClientModelTaskService) // GetModelTaskStatService get model task stat service GetModelTaskStatService() (modelTaskStatSvc GrpcClientModelTaskStatService) // GetNodeConfigService get node config service GetNodeConfigService() (cfgSvc NodeConfigService) // GetCurrentNode get node of the handler GetCurrentNode() (n Node, err error) // GetTaskById get task by id GetTaskById(id primitive.ObjectID) (t Task, err error) // GetSpiderById get task by id GetSpiderById(id primitive.ObjectID) (t Spider, err error) }
type TaskRunner ¶
type TaskSchedulerService ¶
type TaskSchedulerService interface { TaskBaseService // Enqueue task into the task queue Enqueue(t Task) (err error) // DequeueAndSchedule continuously dequeue task and schedule to corresponding node DequeueAndSchedule() // Dequeue task with node info from the task queue Dequeue() (tasks []Task, err error) // Schedule task to corresponding node Schedule(tasks []Task) (err error) // Cancel task to corresponding node Cancel(id primitive.ObjectID) (err error) // SetInterval set the interval or duration between two adjacent fetches SetInterval(interval time.Duration) }
type TaskStat ¶
type TaskStat interface { Model GetCreateTs() (ts time.Time) SetCreateTs(ts time.Time) GetStartTs() (ts time.Time) SetStartTs(ts time.Time) GetEndTs() (ts time.Time) SetEndTs(ts time.Time) GetWaitDuration() (d int64) SetWaitDuration(d int64) GetRuntimeDuration() (d int64) SetRuntimeDuration(d int64) GetTotalDuration() (d int64) SetTotalDuration(d int64) GetResultCount() (c int64) SetResultCount(c int64) GetErrorLogCount() (c int64) SetErrorLogCount(c int64) }
type TaskStatsService ¶
type UserCreateOptions ¶
type UserLoginOptions ¶
type UserService ¶
type UserService interface { Init() (err error) SetJwtSecret(secret string) SetJwtSigningMethod(method jwt.SigningMethod) Create(opts *UserCreateOptions) (err error) Login(opts *UserLoginOptions) (token string, u User, err error) CheckToken(token string) (u User, err error) ChangePassword(id primitive.ObjectID, password string) (err error) MakeToken(user User) (tokenStr string, err error) GetCurrentUser(c *gin.Context) (u User, err error) }
type WithAddress ¶
type WithConfigPath ¶
type WithModelId ¶
Source Files ¶
- address.go
- color.go
- color_service.go
- entity.go
- fs_file_info.go
- fs_service.go
- fs_service_options.go
- grpc_base.go
- grpc_base_service_params.go
- grpc_client.go
- grpc_client_model_base_service.go
- grpc_client_model_delegate.go
- grpc_client_model_node_service.go
- grpc_client_model_service.go
- grpc_client_model_spider_service.go
- grpc_client_model_task_service.go
- grpc_client_model_task_stat_service.go
- grpc_client_pool.go
- grpc_model_base_service_message.go
- grpc_model_binder.go
- grpc_model_delegate_message.go
- grpc_model_list_binder.go
- grpc_server.go
- grpc_subscribe.go
- injectable.go
- list.go
- model.go
- model_artifact.go
- model_artifact_sys.go
- model_base_service.go
- model_binder.go
- model_delegate.go
- model_list_binder.go
- model_node.go
- model_node_delegate.go
- model_result.go
- model_schedule.go
- model_spider.go
- model_tag.go
- model_task.go
- model_task_stat.go
- model_user.go
- module.go
- node_config_service.go
- node_master_service.go
- node_service.go
- node_service_option.go
- node_worker_service.go
- options.go
- plugin_service.go
- provide.go
- result_service.go
- schedule_service.go
- spider_admin_service.go
- spider_fs_service.go
- spider_service_options.go
- spider_sync_service.go
- stats_service.go
- task_base_service.go
- task_handler_service.go
- task_runner.go
- task_scheduler_service.go
- task_stats_service.go
- test.go
- user_service.go
- user_service_options.go
- with_address.go
- with_config_path.go
- with_model_id.go
Click to show internal directories.
Click to hide internal directories.