article_spider

Nov 8, 2023

Changes in this version

type FieldTypes

+ const Attrs

Nov 3, 2023

Changes in this version

type Form

+ ListGetErrorFunc func(form *Form, err error)

Oct 8, 2023

Changes in this version

type Form

+ DisableMessage bool

Oct 8, 2023

Changes in this version

type Form

+ ProxyFinishFunc func(Proxy)

+ type Proxy struct

+ Expire int64

+ Host string

+ Port int

+ Scheme string

Oct 8, 2023

Changes in this version

type Form

+ ProxyFunc func() (string, error)

Sep 25, 2023

Changes in this version

type Form

+ DetailSleep time.Duration

+ ListSleep time.Duration

Sep 22, 2023

Changes in this version

type Form

+ ListUrlCurrent string

Sep 21, 2023

Changes in this version

type Form

+ PageCurrent int

Feb 14, 2023

Changes in this version

type Form

+ Filter func(map[string]string) bool

Oct 9, 2022

Changes in this version

type Field

+ PrefixNotSelector []string

Sep 30, 2022

Changes in this version

type Field

+ ImageResizePercent int

type Spider

+ CustomDownloadFun func(imgUrl string, f *Form, item Field) error

+ func (s *Spider) CustomDownloadImage(fun func(imgUrl string, f *Form, item Field) error) *Spider

Aug 23, 2022

Changes in this version

type Mode

+ func (m Mode) ToString() string

Jul 20, 2022

Changes in this version

+ func NewUrl(s *Spider) *url

type Form

+ DetailUrls []string

type Mode

+ const Url

Jun 6, 2022

Changes in this version

type FieldTypes

+ const File

type Form

+ FilterError bool

+ type Rows struct

+ func NewRows(m map[string]string) *Rows

+ func (r *Rows) Add(rr *Rows)

May 16, 2022

Changes in this version

type Spider

+ func (s *Spider) SetImageDir(path string)

+ func (s *Spider) SetSavePath(path string)

May 5, 2022

Changes in this version

+ const Debug

+ const Error

+ const Finish

+ const Info

+ const Log

+ const Process

+ func If(condition bool, trueVal, falseVal interface{}) interface

+ func NewApi(s *Spider) *api

+ func NewAuto(s *Spider) *auto

+ func NewExcel(s *Spider) *excel

+ func NewNormal(s *Spider) *normal

+ func NewResult(s *Spider) *result

+ type Field struct

+ AttrKey string

+ ConversionFunc func(data string, resList map[string]string) string

+ ExcelHeader string

+ ImageDir string

+ ImagePrefix func(form *Form, imageName string) string

+ LazyImageAttrName string

+ NotSelector []string

+ RegularIndex int

+ Selector string

+ Types FieldTypes

+ type FieldTypes int

+ const Attr

+ const Fixed

+ const HtmlWithImage

+ const Image

+ const MultipleImages

+ const OnlyHtml

+ const Regular

+ const Text

+ type Form struct

+ AllowImageExtension []string

+ ApiConversion func(html string, form *Form) []string

+ AutoDetailForceNewTab bool

+ AutoDetailWaitSelector string

+ AutoListWaitSelector string

+ AutoNextPageMode NextPageMode

+ AutoNextSelector string

+ AutoPrefixEvent func(chromedpCtx context.Context)

+ Channel string

+ ChannelFunc func(form *Form) []string

+ CustomExcelHeader bool

+ DefaultImg func(form *Form, item Field) string

+ DetailCoroutineNumber int

+ DetailFields map[string]Field

+ DisableAutoCoding bool

+ DisableImageExtensionCheck bool

+ Host string

+ HrefSelector string

+ HttpHeader map[string]string

+ HttpProxy string

+ HttpTimeout time.Duration

+ LazyImageAttrName string

+ Length int

+ ListFields map[string]Field

+ ListSelector string

+ MiddleSelector []string

+ PageStart int

+ ResultCallback func(item map[string]string, form *Form)

+ func (f *Form) DealCoding(html string, header http2.Header) (string, error)

+ func (f *Form) DownImg(url string, item Field, res *sync.Map) string

+ func (f *Form) GetCharsetByContentType(contentType string) string

+ func (f *Form) GetDir(path string, res *sync.Map) string

+ func (f *Form) GetHref(href string) string

+ func (f *Form) GetHtml(url string) (string, error)

+ func (f *Form) ResolveSelector(html string, selector map[string]Field, originUrl string) (map[string]string, error)

+ type Mode int

+ const Api

+ const Auto

+ const Normal

+ type NextPageMode int

+ const LoadMore

+ const Pagination

+ type Notice struct

+ func NewNotice(s *Spider) *Notice

+ func (n *Notice) Debug(content ...interface{})

+ func (n *Notice) Error(content ...interface{})

+ func (n *Notice) Finish(content ...interface{})

+ func (n *Notice) Info(content ...interface{})

+ func (n *Notice) Log(content ...interface{})

+ func (n *Notice) Process(content ...interface{})

+ func (n *Notice) Service()

+ func (n *Notice) Stop()

+ type Spider struct

+ func NewSpider(f Form, mode Mode, cxt context.Context) *Spider

+ func (s *Spider) Debug() *Spider

+ func (s *Spider) Start() error