Versions in this module Expand all Collapse all v1 v1.0.0 May 11, 2018 Changes in this version + var PageReqType_name = map[int32]string + var PageReqType_value = map[string]int32 + var SubType_name = map[int32]string + var SubType_value = map[string]int32 + func RegisterIdeaCrawlerServer(s *grpc.Server, srv IdeaCrawlerServer) + type DomainOpt struct + CallbackUrlRegexp string + CallbackXpathMatch []*KVP + CallbackXpathRegexp []*KVP + CancelOnDisconnect bool + CheckContent bool + CheckLoginAfterEachPage bool + Chrome bool + ChromeBinary string + Depth int32 + DomLoadTime int32 + DomainDropPriority bool + DropDomains []string + Firstrun *google_protobuf.Timestamp + FollowOtherDomains bool + FollowUrlRegexp string + Frequency *google_protobuf1.Duration + Impolite bool + KeepDomains []string + Login bool + LoginJS string + LoginParseFields bool + LoginParseXpath []*KVP + LoginPayload []*KVP + LoginSuccessCheck *KVP + LoginUrl string + LoginUsingSelenium bool + MaxConcurrentRequests int32 + MaxDelay int32 + MaxIdleTime int32 + MinDelay int32 + NetworkIface string + NoFollow bool + Prefetch bool + Repeat bool + SeedUrl string + UnsafeNormalizeURL bool + Useragent string + func (*DomainOpt) Descriptor() ([]byte, []int) + func (*DomainOpt) ProtoMessage() + func (m *DomainOpt) GetCallbackUrlRegexp() string + func (m *DomainOpt) GetCallbackXpathMatch() []*KVP + func (m *DomainOpt) GetCallbackXpathRegexp() []*KVP + func (m *DomainOpt) GetCancelOnDisconnect() bool + func (m *DomainOpt) GetCheckContent() bool + func (m *DomainOpt) GetCheckLoginAfterEachPage() bool + func (m *DomainOpt) GetChrome() bool + func (m *DomainOpt) GetChromeBinary() string + func (m *DomainOpt) GetDepth() int32 + func (m *DomainOpt) GetDomLoadTime() int32 + func (m *DomainOpt) GetDomainDropPriority() bool + func (m *DomainOpt) GetDropDomains() []string + func (m *DomainOpt) GetFirstrun() *google_protobuf.Timestamp + func (m *DomainOpt) GetFollowOtherDomains() bool + func (m *DomainOpt) GetFollowUrlRegexp() string + func (m *DomainOpt) GetFrequency() *google_protobuf1.Duration + func (m *DomainOpt) GetImpolite() bool + func (m *DomainOpt) GetKeepDomains() []string + func (m *DomainOpt) GetLogin() bool + func (m *DomainOpt) GetLoginJS() string + func (m *DomainOpt) GetLoginParseFields() bool + func (m *DomainOpt) GetLoginParseXpath() []*KVP + func (m *DomainOpt) GetLoginPayload() []*KVP + func (m *DomainOpt) GetLoginSuccessCheck() *KVP + func (m *DomainOpt) GetLoginUrl() string + func (m *DomainOpt) GetLoginUsingSelenium() bool + func (m *DomainOpt) GetMaxConcurrentRequests() int32 + func (m *DomainOpt) GetMaxDelay() int32 + func (m *DomainOpt) GetMaxIdleTime() int32 + func (m *DomainOpt) GetMinDelay() int32 + func (m *DomainOpt) GetNetworkIface() string + func (m *DomainOpt) GetNoFollow() bool + func (m *DomainOpt) GetPrefetch() bool + func (m *DomainOpt) GetRepeat() bool + func (m *DomainOpt) GetSeedUrl() string + func (m *DomainOpt) GetUnsafeNormalizeURL() bool + func (m *DomainOpt) GetUseragent() string + func (m *DomainOpt) Reset() + func (m *DomainOpt) String() string + type IdeaCrawlerClient interface + AddDomainAndListen func(ctx context.Context, in *DomainOpt, opts ...grpc.CallOption) (IdeaCrawler_AddDomainAndListenClient, error) + AddPages func(ctx context.Context, opts ...grpc.CallOption) (IdeaCrawler_AddPagesClient, error) + CancelJob func(ctx context.Context, in *Subscription, opts ...grpc.CallOption) (*Status, error) + func NewIdeaCrawlerClient(cc *grpc.ClientConn) IdeaCrawlerClient + type IdeaCrawlerServer interface + AddDomainAndListen func(*DomainOpt, IdeaCrawler_AddDomainAndListenServer) error + AddPages func(IdeaCrawler_AddPagesServer) error + CancelJob func(context.Context, *Subscription) (*Status, error) + type IdeaCrawler_AddDomainAndListenClient interface + Recv func() (*PageHTML, error) + type IdeaCrawler_AddDomainAndListenServer interface + Send func(*PageHTML) error + type IdeaCrawler_AddPagesClient interface + CloseAndRecv func() (*Status, error) + Send func(*PageRequest) error + type IdeaCrawler_AddPagesServer interface + Recv func() (*PageRequest, error) + SendAndClose func(*Status) error + type KVP struct + Key string + Value string + func (*KVP) Descriptor() ([]byte, []int) + func (*KVP) ProtoMessage() + func (m *KVP) GetKey() string + func (m *KVP) GetValue() string + func (m *KVP) Reset() + func (m *KVP) String() string + type PageHTML struct + Content []byte + Error string + Httpstatuscode int32 + MetaStr string + Sub *Subscription + Success bool + Url string + func (*PageHTML) Descriptor() ([]byte, []int) + func (*PageHTML) ProtoMessage() + func (m *PageHTML) GetContent() []byte + func (m *PageHTML) GetError() string + func (m *PageHTML) GetHttpstatuscode() int32 + func (m *PageHTML) GetMetaStr() string + func (m *PageHTML) GetSub() *Subscription + func (m *PageHTML) GetSuccess() bool + func (m *PageHTML) GetUrl() string + func (m *PageHTML) Reset() + func (m *PageHTML) String() string + type PageReqType int32 + const PageReqType_BUILTINJS + const PageReqType_GET + const PageReqType_HEAD + const PageReqType_JSCRIPT + func (PageReqType) EnumDescriptor() ([]byte, []int) + func (x PageReqType) String() string + type PageRequest struct + Js string + MetaStr string + NoCallback bool + Reqtype PageReqType + Sub *Subscription + Url string + func (*PageRequest) Descriptor() ([]byte, []int) + func (*PageRequest) ProtoMessage() + func (m *PageRequest) GetJs() string + func (m *PageRequest) GetMetaStr() string + func (m *PageRequest) GetNoCallback() bool + func (m *PageRequest) GetReqtype() PageReqType + func (m *PageRequest) GetSub() *Subscription + func (m *PageRequest) GetUrl() string + func (m *PageRequest) Reset() + func (m *PageRequest) String() string + type Status struct + Error string + Success bool + func (*Status) Descriptor() ([]byte, []int) + func (*Status) ProtoMessage() + func (m *Status) GetError() string + func (m *Status) GetSuccess() bool + func (m *Status) Reset() + func (m *Status) String() string + type SubType int32 + const SubType_DATETIME + const SubType_SEQNUM + func (SubType) EnumDescriptor() ([]byte, []int) + func (x SubType) String() string + type Subscription struct + Datetime *google_protobuf.Timestamp + Domainname string + Seqnum int32 + Subcode string + Subtype SubType + func (*Subscription) Descriptor() ([]byte, []int) + func (*Subscription) ProtoMessage() + func (m *Subscription) GetDatetime() *google_protobuf.Timestamp + func (m *Subscription) GetDomainname() string + func (m *Subscription) GetSeqnum() int32 + func (m *Subscription) GetSubcode() string + func (m *Subscription) GetSubtype() SubType + func (m *Subscription) Reset() + func (m *Subscription) String() string
Go to main page