Versions in this module Expand all Collapse all v0 v0.8.0 Jun 24, 2017 Changes in this version + func CreateSnapshot(snapshot *Snapshot) error + func CreateTask(task *Task) error + func DeleteTask(id string) error + func IncrementDomainLinkCount(host string) error + func UpdateTask(task *Task) + type Domain struct + CreateTime *time.Time + Favicon string + Host string + LinksCount int64 + Settings *DomainSetting + UpdateTime *time.Time + func CreateDomain(host string) Domain + func GetDomain(domain string) (Domain, error) + func GetDomainList(from, size int, domain string) (int, []Domain, error) + type DomainSetting struct + type IndexDocument struct + Id string + Index string + Source map[string]interface{} + Type string + type KV struct + Key string + Value []string + type LinkGroup struct + External []PageLink + Internal []PageLink + type PageLink struct + Label string + Url string + type Seed struct + Breadth int + Depth int + Reference string + Url string + func NewTaskSeed(url, ref string, depth int, breadth int) Seed + func TaskSeedFromBytes(b []byte) Seed + func (this Seed) Get(url string) Seed + func (this Seed) GetBytes() ([]byte, error) + func (this Seed) MustGetBytes() []byte + type Snapshot struct + Bold []string + Classifications []string + ContentType string + CreateTime *time.Time + EnrichedFeatures *map[string]interface{} + File string + H1 []string + H2 []string + H3 []string + H4 []string + H5 []string + Hash string + Headers map[string][]string + ID string + Images struct{ ... } + Italic []string + Language string + Links LinkGroup + Metadata *map[string]interface{} + Parameters []KV + Path string + Payload []byte + SimHash string + Size uint64 + StatusCode int + Summary string + Tags []string + TaskID string + Text string + Title string + Url string + Version int + type Task struct + CreateTime *time.Time + Host string + ID string + LastCheckTime *time.Time + LastFetchTime *time.Time + Message string + NextCheckTime *time.Time + OriginalUrl string + Phrase pipeline.Phrase + Schema string + SnapshotCreateTime *time.Time + SnapshotHash string + SnapshotID string + SnapshotSimHash string + SnapshotVersion int + Status TaskStatus + UpdateTime *time.Time + func GetPendingNewFetchTasks() (int, []Task, error) + func GetPendingUpdateFetchTasks(offset *time.Time) (int, []Task, error) + func GetTask(id string) (Task, error) + func GetTaskByField(k, v string) (Task, error) + func GetTaskList(from, size int, domain string) (int, []Task, error) + type TaskStatus int + const Task404Ignore + const TaskCreated + const TaskFetchFailed + const TaskFetchSuccess + const TaskFetchTimeout + const TaskRedirectedIgnore
Go to main page