crawler

Documentation

Overview

Package crawler implements web crawler.

Index

• type BBT
  • func NewBBTDB(ctx context.Context, addr, dbName, user, pwd, docusColName string) (b *BBT, err error)
• type Dao
  • func NewDao(ctx context.Context, addr, dbName, user, pwd, docusColName string) (*Dao, error)
  • func (d *Dao) RemoveLegacy(ctx context.Context, updateBefore time.Time) error
  • func (d *Dao) Save(ctx context.Context, title, text, url string) error
  • func (d *Dao) Search(ctx context.Context, text string) (rets []SearchResult, err error)
• type Docu
• type SearchResult
• type Service
  • func NewService(ctx context.Context, addr, dbName, user, pwd, docusColName string) (*Service, error)
  • func (s *Service) CrawlAllPages(ctx context.Context, sitemaps []string) error
  • func (s *Service) Search(ctx context.Context, text string) (rets []SearchResult, err error)

Types

type BBT

type BBT struct {
	// contains filtered or unexported fields
}

BBT database for basebit

func NewBBTDB

func NewBBTDB(ctx context.Context, addr, dbName, user, pwd, docusColName string) (b *BBT, err error)

NewBBTDB new bbt db

type Dao

type Dao struct {
	DB *BBT
}

func NewDao

func NewDao(ctx context.Context, addr, dbName, user, pwd, docusColName string) (*Dao, error)

func (*Dao) RemoveLegacy

func (d *Dao) RemoveLegacy(ctx context.Context, updateBefore time.Time) error

func (*Dao) Save

func (d *Dao) Save(ctx context.Context, title, text, url string) error

func (*Dao) Search

func (d *Dao) Search(ctx context.Context, text string) (rets []SearchResult, err error)

Search search text in title and content

type Docu

type Docu struct {
	CreatedAt time.Time `bson:"created_at" json:"created_at"`
	UpdatedAt time.Time `bson:"updated_at" json:"updated_at"`
	Text      string    `bson:"text" json:"text"`
	Title     string    `bson:"title" json:"title"`
	URL       string    `bson:"url" json:"url"`
}

Docu database for bbt document

type SearchResult

type SearchResult struct {
	Context string `bson:"context" json:"context"`
	URL     string `bson:"url" json:"url"`
	Title   string `bson:"title" json:"title"`

	ID   primitive.ObjectID `bson:"_id,omitempty" json:"-"`
	Text string             `bson:"text" json:"-"`
}

type Service

type Service struct {
	// contains filtered or unexported fields
}

func NewService

func NewService(ctx context.Context, addr, dbName, user, pwd, docusColName string) (*Service, error)

func (*Service) CrawlAllPages

func (s *Service) CrawlAllPages(ctx context.Context, sitemaps []string) error

func (*Service) Search

func (s *Service) Search(ctx context.Context, text string) (rets []SearchResult, err error)