grab

README

grab

非结构式数据抓取服务

core

核心代码

Documentation

Index

• Variables
• func DecodeURL(str string) (string, error)
• func EncodeURL(str string) string
• func ExplainLink(url string) (address, drive, page string)
• func GetGuide(drive string) reader.Guide
• func GetGuidex(drive string) reader.Guide
• func GetReader(drive string) reader.Reader
• func GetReaderx(drive string) reader.Reader
• type Card
• type Catalog
• type DemoItem
  • func ArticleDemoList() []DemoItem
  • func BookDemoList() []DemoItem
  • func RssDemoList() []DemoItem
• type Item
  • func GetAbouts() []Item
  • func GetBooks(url string) []Item
  • func GetChapter(url string) []Item
  • func GetChapters(url string) []Item
  • func GetClassify() []Item
  • func GetDrives() []Item
  • func GetHelps() []Item
  • func GetRanks() []Item
  • func GetResource(url string) []Item
  • func GetResources() []Item
  • func GetTopics() []Item
  • func GetWaitExamineClassify() []Item
• type Link

Variables

var GuideList = map[string]reader.Guide{
	"qidian":      &reader.QidianReader{},
	"zongheng":    &reader.ZonghengReader{},
	"17k":         &reader.SeventeenKReader{},
	"xxsy":        &reader.XxsyReader{},
	"baidu":       &reader.BaiduReader{},
	"hongxiu":     &reader.HongxiuReader{},
	"luoqiu":      &reader.MLuoqiuReader{},
	"booktxt":     &reader.BooktxtReader{},
	"paoshu8":     &reader.Paoshu8Reader{},
	"qkshu6":      &reader.Qkshu6Reader{},
	"shuge":       &reader.ShugeReader{},
	"qu":          &reader.QuReader{},
	"jx":          &reader.JxReader{},
	"uxiaoshuo":   &reader.UxiaoshuoReader{},
	"soe8":        &reader.Soe8Reader{},
	"bxks":        &reader.BxksReader{},
	"xin18":       &reader.Xin18Reader{},
	"bxwx":        &reader.BxwxReader{},
	"biqugeinfo":  &reader.BiqugeinfoReader{},
	"mcmssc":      &reader.McmsscReader{},
	"xs280":       &reader.Xs280Reader{},
	"xbiquge":     &reader.XbiqugeReader{},
	"biquyun":     &reader.BiquyunReader{},
	"r2hm":        &reader.R2hmReader{},
	"manwuyu":     &reader.ManwuyuReader{},
	"manhwa":      &reader.ManhwaReader{},
	"aimeizi5":    &reader.Aimeizi5Reader{},
	"kanmeizi":    &reader.KanmeiziReader{},
	"fuman":       &reader.FumanReader{},
	"weijiaoshou": &reader.WeijiaoshouReader{},
	"haimaoba":    &reader.HaimaobaReader{},
	"ssmh":        &reader.SsmhReader{},
	"hanmanku":    &reader.HanmankuReader{},
	"hanmanwo":    &reader.HanmanwoReader{},
	"laosijixs":   &reader.LaosijixsReader{},
}

GuideList 阅读器匹配组

var ReaderList = map[string]reader.Reader{
	"qidian":      &reader.QidianReader{},
	"zongheng":    &reader.ZonghengReader{},
	"17k":         &reader.SeventeenKReader{},
	"xxsy":        &reader.XxsyReader{},
	"hongxiu":     &reader.HongxiuReader{},
	"luoqiu":      &reader.MLuoqiuReader{},
	"booktxt":     &reader.BooktxtReader{},
	"booktxtnet":  &reader.BooktxtnetReader{},
	"paoshu8":     &reader.Paoshu8Reader{},
	"qkshu6":      &reader.Qkshu6Reader{},
	"shuge":       &reader.ShugeReader{},
	"qu":          &reader.QuReader{},
	"jx":          &reader.JxReader{},
	"uxiaoshuo":   &reader.UxiaoshuoReader{},
	"soe8":        &reader.Soe8Reader{},
	"bxks":        &reader.BxksReader{},
	"bxwx":        &reader.BxwxReader{},
	"xin18":       &reader.Xin18Reader{},
	"biqugeinfo":  &reader.BiqugeinfoReader{},
	"mcmssc":      &reader.McmsscReader{},
	"xs280":       &reader.Xs280Reader{},
	"xbiquge":     &reader.XbiqugeReader{},
	"biquyun":     &reader.BiquyunReader{},
	"book":        &reader.BookReader{},
	"article":     &reader.ArticleReader{},
	"rss":         &reader.RssReader{},
	"blog":        &reader.BlogReader{},
	"learnku":     &reader.LearnkuReader{},
	"github":      &reader.GithubReader{},
	"r2hm":        &reader.R2hmReader{},
	"manwuyu":     &reader.ManwuyuReader{},
	"manhwa":      &reader.ManhwaReader{},
	"aimeizi5":    &reader.Aimeizi5Reader{},
	"kanmeizi":    &reader.KanmeiziReader{},
	"fuman":       &reader.FumanReader{},
	"weijiaoshou": &reader.WeijiaoshouReader{},
	"haimaoba":    &reader.HaimaobaReader{},
	"ssmh":        &reader.SsmhReader{},
	"hanmanku":    &reader.HanmankuReader{},
	"hanmanwo":    &reader.HanmanwoReader{},
	"laosijixs":   &reader.LaosijixsReader{},
}

ReaderList 阅读器匹配组

Functions

func DecodeURL

func DecodeURL(str string) (string, error)

DecodeURL 把url decode

func EncodeURL

func EncodeURL(str string) string

EncodeURL 把url encode

func ExplainLink

func ExplainLink(url string) (address, drive, page string)

ExplainLink 解释链接地址

func GetGuide

func GetGuide(drive string) reader.Guide

GetGuide 获取引导

func GetGuidex

func GetGuidex(drive string) reader.Guide

GetGuidex 获取引导

func GetReader

func GetReader(drive string) reader.Reader

GetReader 获取阅读器

func GetReaderx

func GetReaderx(drive string) reader.Reader

GetReaderx 获取阅读器

Types

type Card

type Card struct {
	Title  string   `json:"title"`  // 标题
	WxTo   string   `json:"wxto"`   // 小程序跳转到目标页
	Intro  string   `json:"intro"`  //介绍
	Type   string   `json:"type"`   // card展示形式 media card text image images link
	Cover  string   `json:"cover"`  // 封面图片
	Images []string `json:"images"` // 图片组效果时图片列表
}

Card 使用卡片代替链接

type Catalog

type Catalog struct {
	Title     string `json:"title"`
	Cards     []Card //`json:"links"`
	SourceURL string `json:"source_url"` // 数据抓取时，统一声明数据来源
	Hash      string `json:"hash"`       // 当前目录 Hash
	Previous  Link   `json:"previous"`   // 如果有上一页
	Next      Link   `json:"next"`       // 如果有下一页
}

* * base.go 放一些基础数据结构类,用于制定内外数据结构

Catalog 第三方资源目录(用户可任意订阅此目录)

type DemoItem

type DemoItem struct {
	Title    string `json:"title"`
	URL      string `json:"url"`
	Category string `json:"category"`
}

DemoItem 示例详细

func ArticleDemoList

func ArticleDemoList() []DemoItem

ArticleDemoList 文章示例列表

func BookDemoList

func BookDemoList() []DemoItem

BookDemoList 小说示例列表

func RssDemoList

func RssDemoList() []DemoItem

RssDemoList Rss示例列表

type Item

type Item struct {
	Title string `json:"title"`
	WxTo  string `json:"wxto"`
	Intro string `json:"intro"`
	Type  string `json:"type"`
}

Item 小程序授受参数明细

func GetAbouts

func GetAbouts() []Item

GetAbouts 获得关于我们的介绍

func GetBooks

func GetBooks(url string) []Item

GetBooks 小说目录列表

func GetChapter

func GetChapter(url string) []Item

GetChapter 小说章节详细

func GetChapters

func GetChapters(url string) []Item

GetChapters 小说章节列表

func GetClassify

func GetClassify() []Item

GetClassify 发现分类

func GetDrives

func GetDrives() []Item

GetDrives 获取所有解释引擎

func GetHelps

func GetHelps() []Item

GetHelps 获取帮助(常见问题)

func GetRanks

func GetRanks() []Item

GetRanks 各类榜单

func GetResource

func GetResource(url string) []Item

GetResource 获得资源详细(分类)

func GetResources

func GetResources() []Item

GetResources 自定义资源列表(支持平台目录)

func GetTopics

func GetTopics() []Item

GetTopics 获取专题列表

func GetWaitExamineClassify

func GetWaitExamineClassify() []Item

GetWaitExamineClassify 获取做给审核检查的class

type Link

type Link struct {
	// Basic
	Title string `json:"title"`
	URL   string `json:"url"`
	WxTo  string `json:"wxto"` // 小程序跳转到目标页
}

Link 链接