pholcus_lib

Documentation

Index

• Variables

Variables

var Ljzgjxt = &Spider{
	Name:        "陆家嘴国际信托",
	Description: "陆家嘴国际信托净值数据 [Auto Page] [http://www.ljzitc.com.cn/news/cpjz/index.html]",

	NotDefaultField: true,

	Namespace: func(*Spider) string {
		return "xintuo"
	},

	SubNamespace: func(self *Spider, dataCell map[string]interface{}) string {
		return "fund_src_nav"
	},

	EnableCookie: false,
	RuleTree: &RuleTree{

		Root: func(ctx *Context) {
			ctx.Aid(map[string]interface{}{"loop": [2]int{1, 2}, "Rule": "生成请求"}, "生成请求")
		},

		Trunk: map[string]*Rule{

			"生成请求": {

				AidFunc: func(ctx *Context, aid map[string]interface{}) interface{} {
					for loop := aid["loop"].([2]int); loop[0] < loop[1]; loop[0]++ {
						ctx.AddQueue(&request.Request{
							Url:  "http://www.ljzitc.com.cn/news/cpjzsy/index.html",
							Rule: aid["Rule"].(string),
						})
					}
					return nil
				},

				ParseFunc: func(ctx *Context) {
					query := ctx.GetDom()
					fmt.Println(query)

					ss := query.Find(".product_box.clearboth")
					fmt.Println(ss)

					ss.Each(func(i int, goq *goquery.Selection) {

						if url, ok := goq.Attr("href"); ok {
							ctx.AddQueue(&request.Request{
								Url:  "http://www.ljzitc.com.cn/" + url,
								Rule: "净值详情",
							})
						}

					})
				},
			},

			"净值详情": {

				ParseFunc: func(ctx *Context) {
					query := ctx.GetDom()

					ss := query.Find("ul").Find("li")
					page := 0

					ss.Each(func(i int, goq *goquery.Selection) {

						mingcheng := goq.Children().Eq(0).Find("a").Text()
						urlink := goq.Children().Eq(4).Find("a")

						page++

						urlink.Each(func(i int, goq *goquery.Selection) {

							if url, ok := goq.Attr("href"); ok {
								ctx.AddQueue(&request.Request{
									Url:  "http://www.dgxt.com/" + url,
									Rule: "获取结果",
									Temp: map[string]interface{}{
										"mingcheng":   mingcheng,
										"level1pages": page,
									},
								})
							}

						})
					})
				},
			},

			"获取结果": {

				ItemFields: []string{
					"基金ID",
					"名称",
					"净值",
					"累计净值",
					"估值日期",
				},
				ParseFunc: func(ctx *Context) {
					query := ctx.GetDom()

					ss := query.Find(".cot_fl ul").Find("li")

					var page1 int
					ctx.GetTemp("level1pages", &page1)

					count := 0

					ss.Each(func(i int, goq *goquery.Selection) {

						divDetail := goq.Children().Eq(0)

						var titleMingCheng string
						ctx.GetTemp("mingcheng", &titleMingCheng)

						mingchen := titleMingCheng
						jingzhi := divDetail.Children().Eq(1).Text()
						leijijingzhi := divDetail.Children().Eq(2).Text()
						guzhiriqi := divDetail.Children().Eq(0).Text()

						count++
						fundID := "XTLUJIAZUI" + "P1" + strconv.Itoa(page1) + "L" + strconv.Itoa(count)

						ctx.Output(map[int]interface{}{
							0: fundID,
							1: mingchen,
							2: jingzhi,
							3: leijijingzhi,
							4: guzhiriqi,
						})

					})
				},
			},
		},
	},
}