caspercloud

package module
v0.0.0-...-2ccf43e Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Dec 18, 2015 License: MIT Imports: 33 Imported by: 1

README

Casper Cloud

Travis Status

基准测试

我们Mock了一个服务他的平均响应时间非常短，他产生的HTML非常简单

<html><head><title>Hello World</title></head><body><h1>Hello World</h1></body></html>

然后我们测试三种下载方式的下载时间

1. curl
2. golang http.Client
3. casper js

结果如下

BenchmarkCurl-4         2000        7960292 ns/op
BenchmarkHttpClient-4   100000      179471 ns/op
BenchmarkCasperJs-4     5           3091975091 ns/op

可以看到，Curl启动进程的时间花费了7ms，http.Client启动进程的时间为0， 而CasperJs启动进程的时间最长，大概需要3s。

但是，如果节省CasperJs打开进程的时间，那么实际下载时间是多少？

我们写了两段js

mock.js

var casper = require("casper").create();
casper.start("http://127.0.0.1:20893/hello", function(){
    console.log(this.getHTML());
});
casper.run();

mock_100.js

var casper = require("casper").create();
casper.start("http://127.0.0.1:20893/hello", function(){
    console.log(this.getHTML());
}).repeat(99, function(){
    casper.thenOpen("http://127.0.0.1:20893/hello", function(){
        console.log(this.getHTML());
    });
});
casper.run();

在mock.js中，我们打开一次casperjs只下载一个链接，而在mock_100.js中，我们打开一次下载100个链接。然后我们在服务端让程序10ms返回。再做一次测试，结果如下：

BenchmarkCasperJs-4        20   2824344729 ns/op
BenchmarkCasperJs100-4         5    9634887107 ns/op

可以看到，在批量下载的程序里，一次下载的时间是 (9635 - 2824) / 99 = 68ms

同样的，我们也对比一下curl的单次下载和批量下载，结果如下

BenchmarkCurl-4     2000      22667633 ns/op
BenchmarkCurl100-4        30    1202024886 ns/op

可以看到，curl的一次下载时间是 (1202 - 22) / 99 = 11ms

从而可以看到，在单次下载中，casperjs的耗时是curl的6倍左右。

Documentation

Overview

Package parser is a generated protocol buffer package.

It is generated from these files: 

parser.proto

It has these top-level messages: 

ParseRequest
ParseReply

Index

Constants

View Source 
const (
	PARAM_USERNAME    = "username"
	PARAM_PASSWORD    = "password"
	PARAM_PASSWORD2   = "password2"
	PARAM_VERIFY_CODE = "randcode"
	PARAM_PHONE_NUM   = "phone"

	FAIL                  = "fail"
	NEED_PARAM            = "need_param"
	NOT_SUPPORT           = "not_support"
	WRONG_PASSWORD        = "wrong_password"
	WRONG_VERIFYCODE      = "wrong_verifycode"
	WRONG_SECOND_PASSWORD = "wrong_second_password"
	LOGIN_SUCCESS         = "login_success"
	BEGIN_FETCH_DATA      = "begin_fetch_data"
	FINISH_FETCH_DATA     = "finish_fetch_data"
	FINISH_ALL            = "finish_all"
	OUTPUT_PUBLICKEY      = "output_publickey"
	OUTPUT_VERIFYCODE     = "output_verifycode"
	TAOBAO_FAIL           = "taobao_crawl_failed"
)

Variables

View Source 
var ParseRequestType_name = map[int32]string{
	0: "Eml",
	1: "Html",
	2: "Honeycomb",
	3: "TaobaoShop",
}
View Source 
var ParseRequestType_value = map[string]int32{
	"Eml":        0,
	"Html":       1,
	"Honeycomb":  2,
	"TaobaoShop": 3,
}

Functions

func DecodePassword 

func DecodePassword(p string, privateKey *rsa.PrivateKey) string

func GenerateRSAKey 

func GenerateRSAKey() (*rsa.PrivateKey, error)

func LoadDownloads 

func LoadDownloads(fs []string)

func PKCS1Pad2Encrypt 

func PKCS1Pad2Encrypt(s []byte, pub *rsa.PublicKey) (string, error)

func ParseFile 

func ParseFile(fn string) error

func PostDataToSlack 

func PostDataToSlack(msg, channel string) bool

func PrivateKeyString 

func PrivateKeyString(key *rsa.PrivateKey) []byte

func PublicKeyString 

func PublicKeyString(key *rsa.PublicKey) []byte

func RegisterParserServer 

func RegisterParserServer(s *grpc.Server, srv ParserServer)

func UploadImage 

func UploadImage(path string) string

Types

type Analyzer 

type Analyzer struct {
	ServerList []string `json:"server_list"`
	// contains filtered or unexported fields
}

func NewAnalyzer 

func NewAnalyzer(path string) *Analyzer

func (*Analyzer) Process 

func (p *Analyzer) Process(req *ParseRequest, downloads []string) bool

func (*Analyzer) ProcessZip 

func (p *Analyzer) ProcessZip(req *ParseRequest, body []byte) bool

func (*Analyzer) SendReq 

func (p *Analyzer) SendReq(req *ParseRequest) bool

type CasperCmd 

type CasperCmd struct {
	// contains filtered or unexported fields
}

func (*CasperCmd) Close 

func (self *CasperCmd) Close() bool

func (*CasperCmd) Finished 

func (self *CasperCmd) Finished() bool

func (*CasperCmd) GetArgsValue 

func (self *CasperCmd) GetArgsValue(key string) string

func (*CasperCmd) GetId 

func (self *CasperCmd) GetId() string

func (*CasperCmd) GetMessage 

func (self *CasperCmd) GetMessage() *Output

func (*CasperCmd) GetParseReq 

func (self *CasperCmd) GetParseReq(fetchStatus string) *ParseRequest

func (*CasperCmd) SetInputArgs 

func (self *CasperCmd) SetInputArgs(input map[string]string)

func (*CasperCmd) Successed 

func (self *CasperCmd) Successed() bool

type CasperCmdFactory 

type CasperCmdFactory struct{}

func (*CasperCmdFactory) CreateCommand 

func (s *CasperCmdFactory) CreateCommand(params url.Values, globalContext map[string]interface{}) Command

func (*CasperCmdFactory) CreateCommandWithPrivateKey 

func (s *CasperCmdFactory) CreateCommandWithPrivateKey(params url.Values, globalContext map[string]interface{}, pk *rsa.PrivateKey) Command

type CasperOutput 

type CasperOutput struct {
	Downloads []string `json:"downloads"`
	Mails     []Mail   `json:"mails"`
	Status    string   `json:"status"`
}

type CasperServer 

type CasperServer struct {
	// contains filtered or unexported fields
}

func NewCasperServer 

func NewCasperServer(cf CommandFactory) *CasperServer

func (*CasperServer) GetContext 

func (self *CasperServer) GetContext() map[string]interface{}

func (*CasperServer) Process 

func (self *CasperServer) Process(params url.Values) *Output

func (*CasperServer) ServeHTTP 

func (self *CasperServer) ServeHTTP(w http.ResponseWriter, req *http.Request)

type Command 

type Command interface {
	GetMessage() *Output
	SetInputArgs(map[string]string)
	Finished() bool
	Successed() bool
	GetId() string
	Close() bool
}

type CommandCache 

type CommandCache struct {
	// contains filtered or unexported fields
}

func NewCommandCache 

func NewCommandCache() *CommandCache

func (*CommandCache) Delete 

func (self *CommandCache) Delete(id string)

func (*CommandCache) GetCommand 

func (self *CommandCache) GetCommand(id string) Command

func (*CommandCache) SetCommand 

func (self *CommandCache) SetCommand(c Command)

type CommandFactory 

type CommandFactory interface {
	CreateCommand(url.Values, map[string]interface{}) Command
	CreateCommandWithPrivateKey(url.Values, map[string]interface{}, *rsa.PrivateKey) Command
}

type Mail 

type Mail struct {
	From  string `json:"from"`
	Title string `json:"title"`
}

type Output 

type Output struct {
	Status    string `json:"status"`
	NeedParam string `json:"need_param"`
	Id        string `json:"id"`
	Data      string `json:"data"`
}

type ParseReply 

type ParseReply struct {
	ServerName       string `protobuf:"bytes,1,opt" json:"ServerName,omitempty"`
	IsParsedSucced   bool   `protobuf:"varint,2,opt" json:"IsParsedSucced,omitempty"`
	WriteHbaseSucced bool   `protobuf:"varint,3,opt" json:"WriteHbaseSucced,omitempty"`
}

func (*ParseReply) ProtoMessage 

func (*ParseReply) ProtoMessage()

func (*ParseReply) Reset 

func (m *ParseReply) Reset()

func (*ParseReply) String 

func (m *ParseReply) String() string

type ParseRequest 

type ParseRequest struct {
	ClientName  string           `protobuf:"bytes,1,opt" json:"ClientName,omitempty"`
	Tmpl        string           `protobuf:"bytes,2,opt" json:"Tmpl,omitempty"`
	RowKey      string           `protobuf:"bytes,3,opt" json:"RowKey,omitempty"`
	UserName    string           `protobuf:"bytes,4,opt" json:"UserName,omitempty"`
	PublicKey   string           `protobuf:"bytes,5,opt" json:"PublicKey,omitempty"`
	PrivateKey  string           `protobuf:"bytes,6,opt" json:"PrivateKey,omitempty"`
	Secret      string           `protobuf:"bytes,7,opt" json:"Secret,omitempty"`
	IsZip       bool             `protobuf:"varint,8,opt" json:"IsZip,omitempty"`
	Data        []string         `protobuf:"bytes,9,rep" json:"Data,omitempty"`
	ParsedData  string           `protobuf:"bytes,10,opt" json:"ParsedData,omitempty"`
	ReqType     ParseRequestType `protobuf:"varint,11,opt,enum=parser.ParseRequestType" json:"ReqType,omitempty"`
	FetchStatus string           `protobuf:"bytes,12,opt" json:"FetchStatus,omitempty"`
	// DataMetaInfo is responsed for Data, DataMetaInfo[index] maps Data[index]
	DataMetaInfo []string `protobuf:"bytes,13,rep" json:"DataMetaInfo,omitempty"`
}

func (*ParseRequest) ProtoMessage 

func (*ParseRequest) ProtoMessage()

func (*ParseRequest) Reset 

func (m *ParseRequest) Reset()

func (*ParseRequest) String 

func (m *ParseRequest) String() string

type ParseRequestType 

type ParseRequestType int32
const (
	ParseRequestType_Eml              ParseRequestType = 0
	ParseRequestType_Html             ParseRequestType = 1
	ParseRequestType_Honeycomb        ParseRequestType = 2
	ParseRequestType_TaobaoShop       ParseRequestType = 3
	ParseRequestType_Honeycomb_Taobao ParseRequestType = 4
	ParseRequestType_Honeycomb_Ccrc   ParseRequestType = 5
	ParseRequestType_Honeycomb_Phone  ParseRequestType = 6
	ParseRequestType_Default          ParseRequestType = 7
)

func (ParseRequestType) String 

func (x ParseRequestType) String() string

type ParserClient 

type ParserClient interface {
	// Sends a parse request
	ProcessParseRequest(ctx context.Context, in *ParseRequest, opts ...grpc.CallOption) (*ParseReply, error)
}

func NewParserClient 

func NewParserClient(cc *grpc.ClientConn) ParserClient

type ParserServer 

type ParserServer interface {
	// Sends a parse request
	ProcessParseRequest(context.Context, *ParseRequest) (*ParseReply, error)
}

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.