filter

Documentation

Index

• Constants
• Variables
• func MarkPath(path string) string
• type SimpleFilter
  • func (s *SimpleFilter) DoFilter(req *model.Request) bool
  • func (s *SimpleFilter) DomainFilter(req *model.Request) bool
  • func (s *SimpleFilter) StaticFilter(req *model.Request) bool
  • func (s *SimpleFilter) UniqueFilter(req *model.Request) bool
• type SmartFilter
  • func (s *SmartFilter) DoFilter(req *model.Request) bool
  • func (s *SmartFilter) Init()

Constants

const (
	MaxParentPathCount         = 32 // 相对于上一级目录，本级path目录的数量修正最大值
	MaxParamKeySingleCount     = 8  // 某个URL参数名重复修正最大值
	MaxParamKeyAllCount        = 10 // 本轮所有URL中某个参数名的重复修正最大值
	MaxPathParamEmptyCount     = 10 // 某个path下的参数值为空，参数名个数修正最大值
	MaxPathParamKeySymbolCount = 5  // 某个Path下的某个参数的标记数量超过此值，则该参数被全局标记
)

const (
	CustomValueMark    = "{{Crawlergo}}"
	FixParamRepeatMark = "{{fix_param}}"
	FixPathMark        = "{{fix_path}}"
	TooLongMark        = "{{long}}"
	NumberMark         = "{{number}}"
	ChineseMark        = "{{chinese}}"
	UpperMark          = "{{upper}}"
	LowerMark          = "{{lower}}"
	UrlEncodeMark      = "{{urlencode}}"
	UnicodeMark        = "{{unicode}}"
	BoolMark           = "{{bool}}"
	ListMark           = "{{list}}"
	TimeMark           = "{{time}}"
	MixAlphaNumMark    = "{{mix_alpha_num}}"
	MixSymbolMark      = "{{mix_symbol}}"
	MixNumMark         = "{{mix_num}}"
	NoLowerAlphaMark   = "{{no_lower}}"
	MixStringMark      = "{{mix_str}}"
)

Variables

var OneNumberRegex = regexp.MustCompile(`[0-9]`)

Functions

func MarkPath

func MarkPath(path string) string

* 标记路径

Types

type SimpleFilter

type SimpleFilter struct {
	UniqueSet mapset.Set
	HostLimit string
}

func (*SimpleFilter) DoFilter

func (s *SimpleFilter) DoFilter(req *model.Request) bool

* 需要过滤则返回 true

func (*SimpleFilter) DomainFilter

func (s *SimpleFilter) DomainFilter(req *model.Request) bool

* 只保留指定域名的链接

func (*SimpleFilter) StaticFilter

func (s *SimpleFilter) StaticFilter(req *model.Request) bool

* 静态资源过滤

func (*SimpleFilter) UniqueFilter

func (s *SimpleFilter) UniqueFilter(req *model.Request) bool

* 请求去重

type SmartFilter

type SmartFilter struct {
	StrictMode   bool
	SimpleFilter SimpleFilter
	// contains filtered or unexported fields
}

func (*SmartFilter) DoFilter

func (s *SmartFilter) DoFilter(req *model.Request) bool

* 智能去重 可选严格模式

需要过滤则返回 true

func (*SmartFilter) Init

func (s *SmartFilter) Init()