service

Documentation

Index

• type AlodokterCrawler
  • func NewAlodokterService(config config.Configuration, logger logger.Logger, ...) *AlodokterCrawler
  • func (a *AlodokterCrawler) CheckUrlIsExist(dataList []entity.AlodokterPenyakit) (listUrl []entity.AlodokterValidation, err error)
  • func (a *AlodokterCrawler) GetNamaObat() (dataList []entity.AlodokterObat, err error)
  • func (a *AlodokterCrawler) GetNamaPenyakit() (dataList []entity.AlodokterPenyakit, err error)
• type AlodokterCrawlerService
• type BasicCrawling
  • func NewCrawler(config config.Configuration, logger logger.Logger, ...) (c *BasicCrawling)
  • func (c *BasicCrawling) ContainsD(link string) (contains bool, err error)
  • func (c *BasicCrawling) Crawling(seedurl string, task string) (err error)
  • func (c *BasicCrawling) CrawlpageList(param entity.CrawlpageListParam) (dataList []entity.CrawlpageListData, err error)
  • func (c *BasicCrawling) CrawlpageListParsed(param entity.CrawlpageListParam) (dataList []entity.CrawlpageListParsedData, err error)
  • func (c *BasicCrawling) ExtractURL(inputurl string, html string) (filteredHrefs []string, err error)
  • func (c *BasicCrawling) Fetch(url string) (fetchres entity.FetchResult, err error)
  • func (c *BasicCrawling) Fetch2(url string) (htmltext entity.FetchResult, err error)
  • func (c *BasicCrawling) GetExistingQueue() (queue []string, err error)
  • func (c *BasicCrawling) GetLatestSeedUrl(param_seedurl string) (seedurl string, err error)
  • func (c *BasicCrawling) StartCrawling() (err error)
  • func (c *BasicCrawling) StopCrawling() (err error)
  • func (c *BasicCrawling) StoreD(pagesource string, link string, fr entity.FetchResult) (err error)
  • func (c *BasicCrawling) StoreDocument(link string, documentype string, document []byte, documentcontenttype string) (err error)
  • func (c *BasicCrawling) StoreE(link string, href string) (err error)
  • func (c *BasicCrawling) TestCrawling()
• type CrawlerService
• type Queue
  • func NewQueue() *Queue
  • func NewQueueFromExisting(queue []string) *Queue
  • func (q *Queue) Contains(item string) bool
  • func (q *Queue) Dequeue() string
  • func (q *Queue) Enqueue(item string)
  • func (q *Queue) IsEmpty() bool

Types

type AlodokterCrawler

type AlodokterCrawler struct {
	// contains filtered or unexported fields
}

func NewAlodokterService

func NewAlodokterService(config config.Configuration, logger logger.Logger, database datastore.Datastore) *AlodokterCrawler

func (*AlodokterCrawler) CheckUrlIsExist

func (a *AlodokterCrawler) CheckUrlIsExist(dataList []entity.AlodokterPenyakit) (listUrl []entity.AlodokterValidation, err error)

func (*AlodokterCrawler) GetNamaObat

func (a *AlodokterCrawler) GetNamaObat() (dataList []entity.AlodokterObat, err error)

func (*AlodokterCrawler) GetNamaPenyakit

func (a *AlodokterCrawler) GetNamaPenyakit() (dataList []entity.AlodokterPenyakit, err error)

type AlodokterCrawlerService

type AlodokterCrawlerService interface {
	GetNamaPenyakit() (dataList []entity.AlodokterPenyakit, err error)
	GetNamaObat() (dataList []entity.AlodokterObat, err error)
	CheckUrlIsExist(dataList []entity.AlodokterPenyakit) (listUrl []entity.AlodokterValidation, err error)
}

type BasicCrawling

type BasicCrawling struct {

	// SeedURL   string
	Task      string
	Datastore datastore.Datastore
	IsStop    bool
	// contains filtered or unexported fields
}

func NewCrawler

func NewCrawler(config config.Configuration, logger logger.Logger, datastore datastore.Datastore) (c *BasicCrawling)

func (*BasicCrawling) ContainsD

func (c *BasicCrawling) ContainsD(link string) (contains bool, err error)

func (*BasicCrawling) Crawling

func (c *BasicCrawling) Crawling(seedurl string, task string) (err error)

func (*BasicCrawling) CrawlpageList

func (c *BasicCrawling) CrawlpageList(param entity.CrawlpageListParam) (dataList []entity.CrawlpageListData, err error)

func (*BasicCrawling) CrawlpageListParsed

func (c *BasicCrawling) CrawlpageListParsed(param entity.CrawlpageListParam) (dataList []entity.CrawlpageListParsedData, err error)

func (*BasicCrawling) ExtractURL

func (c *BasicCrawling) ExtractURL(inputurl string, html string) (filteredHrefs []string, err error)

func (*BasicCrawling) Fetch

func (c *BasicCrawling) Fetch(url string) (fetchres entity.FetchResult, err error)

func (*BasicCrawling) Fetch2

func (c *BasicCrawling) Fetch2(url string) (htmltext entity.FetchResult, err error)

func (*BasicCrawling) GetExistingQueue

func (c *BasicCrawling) GetExistingQueue() (queue []string, err error)

func (*BasicCrawling) GetLatestSeedUrl

func (c *BasicCrawling) GetLatestSeedUrl(param_seedurl string) (seedurl string, err error)

func (*BasicCrawling) StartCrawling

func (c *BasicCrawling) StartCrawling() (err error)

func (*BasicCrawling) StopCrawling

func (c *BasicCrawling) StopCrawling() (err error)

func (*BasicCrawling) StoreD

func (c *BasicCrawling) StoreD(pagesource string, link string, fr entity.FetchResult) (err error)

func (*BasicCrawling) StoreDocument

func (c *BasicCrawling) StoreDocument(link string, documentype string, document []byte, documentcontenttype string) (err error)

func (*BasicCrawling) StoreE

func (c *BasicCrawling) StoreE(link string, href string) (err error)

func (*BasicCrawling) TestCrawling

func (c *BasicCrawling) TestCrawling()

type CrawlerService

type CrawlerService interface {
	Crawling(seedurl string, task string) (err error)
	TestCrawling()
	StartCrawling() (err error)
	StopCrawling() (err error)
	CrawlpageList(param entity.CrawlpageListParam) (dataList []entity.CrawlpageListData, err error)
	CrawlpageListParsed(param entity.CrawlpageListParam) (dataList []entity.CrawlpageListParsedData, err error)
}

type Queue

type Queue struct {
	Queue []string
}

func NewQueue

func NewQueue() *Queue

NewQueue initializes a new queue

func NewQueueFromExisting

func NewQueueFromExisting(queue []string) *Queue

func (*Queue) Contains

func (q *Queue) Contains(item string) bool

Contains checks if the queue contains a specific item

func (*Queue) Dequeue

func (q *Queue) Dequeue() string

Dequeue removes and returns an item from the queue

func (*Queue) Enqueue

func (q *Queue) Enqueue(item string)

Enqueue adds an item to the queue

func (*Queue) IsEmpty

func (q *Queue) IsEmpty() bool

IsEmpty checks if the queue is empty