sensitive

Documentation

Overview

*

• 敏感词库

Index

• func Load(group string, path string) error
• func LoadUrl(group string, url string) error
• func LoadUrl2(url string) error
• type Filter
  • func Add(group string) *Filter
  • func Get(group string) *Filter
  • func New() *Filter
  • func (filter *Filter) AddWord(words ...string)
  • func (filter *Filter) DelWord(words ...string)
  • func (filter *Filter) Filter(text string) string
  • func (filter *Filter) FindAll(text string) []*Group
  • func (filter *Filter) FindIn(text string) (bool, string)
  • func (filter *Filter) Load(rd io.Reader) error
  • func (filter *Filter) LoadNetWordDict(url string) error
  • func (filter *Filter) LoadWordDict(path string) error
  • func (filter *Filter) RemoveNoise(text string) string
  • func (filter *Filter) Replace(text string, repl rune) string
  • func (filter *Filter) UpdateNoisePattern(pattern string)
  • func (filter *Filter) Validate(text string) (bool, string)
• type Group
• type Node
  • func NewNode(character rune) *Node
  • func NewRootNode(character rune) *Node
  • func (node *Node) IsLeafNode() bool
  • func (node *Node) IsPathEnd() bool
  • func (node *Node) IsRootNode() bool
  • func (node *Node) SoftDel()
• type Trie
  • func NewTrie() *Trie
  • func (tree *Trie) Add(words ...string)
  • func (tree *Trie) Del(words ...string)
  • func (tree *Trie) Filter(text string) string
  • func (tree *Trie) FindAll(text string) []*Group
  • func (tree *Trie) FindIn(text string) (bool, string)
  • func (tree *Trie) Replace(text string, character rune) string
  • func (tree *Trie) Validate(text string) (bool, string)

Functions

func Load

func Load(group string, path string) error

func LoadUrl

func LoadUrl(group string, url string) error

func LoadUrl2

func LoadUrl2(url string) error

*

data: {"code":200,"data":{"vibo_title":[],"":[]}}

Types

type Filter

type Filter struct {
	// contains filtered or unexported fields
}

Filter 敏感词过滤器

func Add

func Add(group string) *Filter

func Get

func Get(group string) *Filter

func New

func New() *Filter

New 返回一个敏感词过滤器

func (*Filter) AddWord

func (filter *Filter) AddWord(words ...string)

AddWord 添加敏感词

func (*Filter) DelWord

func (filter *Filter) DelWord(words ...string)

DelWord 删除敏感词

func (*Filter) Filter

func (filter *Filter) Filter(text string) string

Filter 过滤敏感词

func (*Filter) FindAll

func (filter *Filter) FindAll(text string) []*Group

FindAll 找到所有匹配词

func (*Filter) FindIn

func (filter *Filter) FindIn(text string) (bool, string)

FindIn 检测敏感词

func (*Filter) Load

func (filter *Filter) Load(rd io.Reader) error

Load common method to add words

func (*Filter) LoadNetWordDict

func (filter *Filter) LoadNetWordDict(url string) error

LoadNetWordDict 加载网络敏感词字典

func (*Filter) LoadWordDict

func (filter *Filter) LoadWordDict(path string) error

LoadWordDict 加载敏感词字典

func (*Filter) RemoveNoise

func (filter *Filter) RemoveNoise(text string) string

RemoveNoise 去除空格等噪音

func (*Filter) Replace

func (filter *Filter) Replace(text string, repl rune) string

Replace 和谐敏感词

func (*Filter) UpdateNoisePattern

func (filter *Filter) UpdateNoisePattern(pattern string)

UpdateNoisePattern 更新去噪模式

func (*Filter) Validate

func (filter *Filter) Validate(text string) (bool, string)

Validate 检测字符串是否合法

type Group

type Group struct {
	Text  string `xml:"text" json:"text"`
	Count int    `xml:"count" json:"count"`
	Index []int  `xml:"index" json:"index"`
}

type Node

type Node struct {
	Character rune
	Children  map[rune]*Node
	// contains filtered or unexported fields
}

Node Trie树上的一个节点.

func NewNode

func NewNode(character rune) *Node

NewNode 新建子节点

func NewRootNode

func NewRootNode(character rune) *Node

NewRootNode 新建根节点

func (*Node) IsLeafNode

func (node *Node) IsLeafNode() bool

IsLeafNode 判断是否叶子节点

func (*Node) IsPathEnd

func (node *Node) IsPathEnd() bool

IsPathEnd 判断是否为某个路径的结束

func (*Node) IsRootNode

func (node *Node) IsRootNode() bool

IsRootNode 判断是否为根节点

func (*Node) SoftDel

func (node *Node) SoftDel()

SoftDel 置软删除状态

type Trie

type Trie struct {
	Root *Node
}

Trie 短语组成的Trie树.

func NewTrie

func NewTrie() *Trie

NewTrie 新建一棵Trie

func (*Trie) Add

func (tree *Trie) Add(words ...string)

Add 添加若干个词

func (*Trie) Del

func (tree *Trie) Del(words ...string)

func (*Trie) Filter

func (tree *Trie) Filter(text string) string

Filter 直接过滤掉字符串中的敏感词

func (*Trie) FindAll

func (tree *Trie) FindAll(text string) []*Group

FindAll 找有所有包含在词库中的词

func (*Trie) FindIn

func (tree *Trie) FindIn(text string) (bool, string)

FindIn 判断text中是否含有词库中的词

func (*Trie) Replace

func (tree *Trie) Replace(text string, character rune) string

Replace 词语替换

func (*Trie) Validate

func (tree *Trie) Validate(text string) (bool, string)

Validate 验证字符串是否合法，如不合法则返回false和检测到 的第一个敏感词