gs

Documentation

Index

• func IsTemporary(err error) bool
• type CollectHandle
• type Collector
  • func NewCollector(proxySock5Host, proxyControllerHost string, collect CollectHandle, ...) *Collector
  • func (c *Collector) RefreshClient() error
• type Config
• type Raw
• type Repository
  • func NewRepository(dsn string, logLevel logger.LogLevel) *Repository
  • func (r *Repository) AddTasks(ctx context.Context, tasks []Task) (uint64, error)
  • func (r *Repository) AddToUpdateChannel(task Task)
  • func (r *Repository) ConsumePendingTasks(ctx context.Context, limit uint32) ([]Task, error)
  • func (r *Repository) ConsumeRaw(id uint64)
  • func (r *Repository) ConsumeRaws(list []uint64)
  • func (r *Repository) DeleteTask(ctx context.Context, id uint64)
  • func (r *Repository) ListRaws(ctx context.Context, tag string, limit uint32) ([]Raw, error)
  • func (r *Repository) SaveRaw(tag string, url string, data []byte) error
• type Server
  • func NewDefaultServer() *Server
  • func NewServer(cfg *Config) *Server
  • func (s *Server) AddTasks(ctx context.Context, req *api.AddTasksReq) (*api.OperationResp, error)
  • func (s *Server) ConsumeRaws(_ context.Context, req *api.ConsumeRawsReq) (*api.OperationResp, error)
  • func (s *Server) ListRaws(ctx context.Context, req *api.ListRawsReq) (*api.RawsResp, error)
  • func (s *Server) Run()
  • func (s *Server) ServerMonitor()
• type Task
• type TemporaryError
  • func (t *TemporaryError) Temporary() bool

Functions

func IsTemporary

func IsTemporary(err error) bool

Types

type CollectHandle

type CollectHandle func(collector *Collector, targetURL string) ([]byte, error)

type Collector

type Collector struct {
	// contains filtered or unexported fields
}

func NewCollector

func NewCollector(proxySock5Host, proxyControllerHost string, collect CollectHandle, concurrency uint) *Collector

func (*Collector) RefreshClient

func (c *Collector) RefreshClient() error

type Config

type Config struct {
	Debug             bool
	DSN               string
	TorSock5Host      string
	TorControllerHost string
	CollectHandle     CollectHandle
	Concurrency       uint
}

type Raw

type Raw struct {
	ID        uint64         `gorm:"primarykey"`
	CreatedAt time.Time      `json:"created_at" gorm:"index"`
	DeletedAt gorm.DeletedAt `json:"deleted_at" gorm:"index"`
	URL       string         `json:"url" gorm:"index"`
	Tag       string         `json:"tag" gorm:"index"`
	Data      []byte         `json:"data"`
}

type Repository

type Repository struct {
	// contains filtered or unexported fields
}

func NewRepository

func NewRepository(dsn string, logLevel logger.LogLevel) *Repository

func (*Repository) AddTasks

func (r *Repository) AddTasks(ctx context.Context, tasks []Task) (uint64, error)

func (*Repository) AddToUpdateChannel

func (r *Repository) AddToUpdateChannel(task Task)

func (*Repository) ConsumePendingTasks

func (r *Repository) ConsumePendingTasks(ctx context.Context, limit uint32) ([]Task, error)

func (*Repository) ConsumeRaw

func (r *Repository) ConsumeRaw(id uint64)

func (*Repository) ConsumeRaws

func (r *Repository) ConsumeRaws(list []uint64)

func (*Repository) DeleteTask

func (r *Repository) DeleteTask(ctx context.Context, id uint64)

func (*Repository) ListRaws

func (r *Repository) ListRaws(ctx context.Context, tag string, limit uint32) ([]Raw, error)

func (*Repository) SaveRaw

func (r *Repository) SaveRaw(tag string, url string, data []byte) error

type Server

type Server struct {
	// contains filtered or unexported fields
}

func NewDefaultServer

func NewDefaultServer() *Server

func NewServer

func NewServer(cfg *Config) *Server

func (*Server) AddTasks

func (s *Server) AddTasks(ctx context.Context, req *api.AddTasksReq) (*api.OperationResp, error)

func (*Server) ConsumeRaws

func (s *Server) ConsumeRaws(_ context.Context, req *api.ConsumeRawsReq) (*api.OperationResp, error)

func (*Server) ListRaws

func (s *Server) ListRaws(ctx context.Context, req *api.ListRawsReq) (*api.RawsResp, error)

func (*Server) Run

func (s *Server) Run()

func (*Server) ServerMonitor

func (s *Server) ServerMonitor()

type Task

type Task struct {
	ID         uint64    `gorm:"primarykey"`
	CreatedAt  time.Time `json:"created_at" gorm:"index"`
	UpdatedAt  time.Time `json:"updated_at" gorm:"index"`
	URL        string    `json:"url" gorm:"index:unique"`
	Tag        string    `json:"tag" gorm:"index"`
	Next       uint64    `json:"next" gorm:"index"`
	IntervalMS uint64    `json:"interval_ms"`
}

type TemporaryError

type TemporaryError struct {
	// contains filtered or unexported fields
}

func (*TemporaryError) Temporary

func (t *TemporaryError) Temporary() bool