Documentation ¶
Overview ¶
Package interpreter has the utilities and defnition of the interpreter implementation of octopus
Package interpreter has the utilities and defnition of the interpreter implementation of octopus
Index ¶
- Constants
- Variables
- func AddRule(rule Rule, priority, groupPriority int, tag string)
- func Cache(in chan Request)
- func Dictionary(in chan DICTRequest)
- func SendDICTToChannel(ch chan DICTRequest, req DICTRequest)
- func SendTokenizerToChannel(ch chan Request, req Request)
- func SetDefaultDICTAggregator(agg DICTAggregator)
- func SetRuleDisableState(pos, groupPos int, state bool)
- func StartCheckingForDates(sentence []rune) (chan datetime.Results, error)
- type ColumnNode
- func (c *ColumnNode) Copy() Node
- func (c *ColumnNode) ID() string
- func (c *ColumnNode) IsResolved() bool
- func (c *ColumnNode) MarshalJSON() ([]byte, error)
- func (c *ColumnNode) PID() string
- func (c *ColumnNode) Parent() Node
- func (c *ColumnNode) SetResolved(state bool)
- func (c *ColumnNode) TokenWord() []rune
- func (c *ColumnNode) Type() Type
- func (c *ColumnNode) UnmarshalJSON(data []byte) error
- type DICT
- type DICTAggregator
- type DICTRequest
- type DICTRequestType
- type FastToken
- type KMP
- type KnowledgeBaseNode
- func (k *KnowledgeBaseNode) Copy() Node
- func (k *KnowledgeBaseNode) ID() string
- func (k *KnowledgeBaseNode) IsResolved() bool
- func (k *KnowledgeBaseNode) MarshalJSON() ([]byte, error)
- func (k *KnowledgeBaseNode) PID() string
- func (k *KnowledgeBaseNode) Parent() Node
- func (k *KnowledgeBaseNode) SetResolved(state bool)
- func (k *KnowledgeBaseNode) TokenWord() []rune
- func (k *KnowledgeBaseNode) Type() Type
- func (k *KnowledgeBaseNode) UnmarshalJSON(data []byte) error
- type KnowledgeBaseType
- type Node
- type OperatorNode
- func (o *OperatorNode) Copy() Node
- func (o *OperatorNode) ID() string
- func (o *OperatorNode) IsResolved() bool
- func (o *OperatorNode) MarshalJSON() ([]byte, error)
- func (o *OperatorNode) PID() string
- func (o *OperatorNode) Parent() Node
- func (o *OperatorNode) SetResolved(state bool)
- func (o *OperatorNode) TokenWord() []rune
- func (o *OperatorNode) Type() Type
- func (o *OperatorNode) UnmarshalJSON(data []byte) error
- type Query
- type Request
- type RequestType
- type Rule
- type RuleGroup
- type SQLQuery
- type TableNode
- func (t *TableNode) Copy() Node
- func (t *TableNode) ID() string
- func (t *TableNode) IsResolved() bool
- func (t *TableNode) MarshalJSON() ([]byte, error)
- func (t *TableNode) PID() string
- func (t *TableNode) Parent() Node
- func (t *TableNode) SetResolved(state bool)
- func (t *TableNode) TokenWord() []rune
- func (t *TableNode) Type() Type
- func (t *TableNode) UnmarshalJSON(data []byte) error
- type TimeNode
- func (t *TimeNode) Copy() Node
- func (t *TimeNode) ID() string
- func (t *TimeNode) IsResolved() bool
- func (t *TimeNode) MarshalJSON() ([]byte, error)
- func (t *TimeNode) PID() string
- func (t *TimeNode) Parent() Node
- func (t *TimeNode) SetResolved(state bool)
- func (t *TimeNode) TokenWord() []rune
- func (t *TimeNode) Type() Type
- func (t *TimeNode) UnmarshalJSON(data []byte) error
- type Token
- type Tokenizer
- type Type
- type UnknownNode
- func (u *UnknownNode) Copy() Node
- func (u *UnknownNode) ID() string
- func (u *UnknownNode) IsResolved() bool
- func (u *UnknownNode) MarshalJSON() ([]byte, error)
- func (u *UnknownNode) PID() string
- func (u *UnknownNode) Parent() Node
- func (u *UnknownNode) SetResolved(state bool)
- func (u *UnknownNode) TokenWord() []rune
- func (u *UnknownNode) Type() Type
- func (u *UnknownNode) UnmarshalJSON(data []byte) error
- type ValueNode
- func (v *ValueNode) Copy() Node
- func (v *ValueNode) ID() string
- func (v *ValueNode) IsResolved() bool
- func (v *ValueNode) MarshalJSON() ([]byte, error)
- func (v *ValueNode) PID() string
- func (v *ValueNode) Parent() Node
- func (v *ValueNode) SetResolved(state bool)
- func (v *ValueNode) TokenWord() []rune
- func (v *ValueNode) Type() Type
- func (v *ValueNode) UnmarshalJSON(data []byte) error
Constants ¶
const ( //AggregationFnCount for count aggregation funtion AggregationFnCount = "COUNT" //AggregationFnSum for sum aggregation funtion AggregationFnSum = "SUM" //AggregationFnAvg for average aggregation funtion AggregationFnAvg = "AVG" )
const ( //DataTypeInt denotes integer data type DataTypeInt = "INT" //DataTypeFloat denotes float data type DataTypeFloat = "FLOAT" //DataTypeString denotes string data type DataTypeString = "STRING" //DataTypeDate denotes date data type DataTypeDate = "DATE" )
const ContainsOperator = "HAS"
ContainsOperator indicates contains operator
const DICTClearCheckInterval = time.Minute * 20
DICTClearCheckInterval is the interval after which the dict removal check has to run
const DICTExpiry = time.Hour * 4
DICTExpiry is the expiry time after which the dictionary expiries without any active usage
const DefaultAggregationFn = AggregationFnCount
DefaultAggregationFn is the default aggregation function for all the columns
const EqOperator = "="
EqOperator indicates equal to operator
const GreaterOperator = ">="
GreaterOperator indicates greater than operator
const LessOperator = "<="
LessOperator indicates less than or equal to operator
const LikeOperator = "LIKE"
LikeOperator indicates like operator
const NotEqOperator = "<>"
NotEqOperator indicates not equal to operator
Variables ¶
var DICTInputChannel chan DICTRequest
DICTInputChannel is the input channel to communicate with the cache
var TokenizerInputChannel chan Request
TokenizerInputChannel is the input channel to communicate with the cache
Functions ¶
func AddRule ¶
AddRule will add a given rule with the given priority and priority in the group It will also build the kmp pattern for the rule
func Cache ¶
func Cache(in chan Request)
Cache is the cache for providing the tokenizer to the platform on demand
func Dictionary ¶
func Dictionary(in chan DICTRequest)
Dictionary is the cache for providing the dictionary to the platform on demand
func SendDICTToChannel ¶
func SendDICTToChannel(ch chan DICTRequest, req DICTRequest)
SendDICTToChannel sends a dict request to the channel. This function is to be used with go routines so that dictionary isn't blocked by the requests
func SendTokenizerToChannel ¶
SendTokenizerToChannel sends a dict request to the channel. This function is to be used with go routines so that tokenizer isn't blocked by the requests
func SetDefaultDICTAggregator ¶
func SetDefaultDICTAggregator(agg DICTAggregator)
SetDefaultDICTAggregator sets the default aggregator as the passed param
func SetRuleDisableState ¶
SetRuleDisableState will set the disable state of a rule
Types ¶
type ColumnNode ¶
type ColumnNode struct { //UID is the unique id of the column node UID string //Word is the word with which the column node has to be matched Word []rune //PN is the parent node of the column. It will be a table node PN *TableNode //PUID is the UID of column's parent node PUID string //Name is the name of the node Name string //Children are the children node of the column node Children []ValueNode //Resolved indicates that the node is resolved Resolved bool //Dimension indicates that the column can be used as dimension Dimension bool //Measure indicates that the column can be used as measure Measure bool //AggregationFn is preferred aggregation with the column node if used as a measure across a dimension AggregationFn string //DataType of the column DataType string //Description is the description about the column Description string //DateFormat is the format of the data if date type DateFormat string }
ColumnNode is the node storing the information about a column. It can be the child of a Table Node and can have value as children
func (*ColumnNode) IsResolved ¶
func (c *ColumnNode) IsResolved() bool
IsResolved will return true if the node is resolved
func (*ColumnNode) MarshalJSON ¶
func (c *ColumnNode) MarshalJSON() ([]byte, error)
MarshalJSON encodes the node into a serializable json
func (*ColumnNode) SetResolved ¶
func (c *ColumnNode) SetResolved(state bool)
SetResolved will set the resolved state of the node
func (*ColumnNode) TokenWord ¶
func (c *ColumnNode) TokenWord() []rune
TokenWord returns the word property of the node
func (*ColumnNode) UnmarshalJSON ¶
func (c *ColumnNode) UnmarshalJSON(data []byte) error
UnmarshalJSON decodes the node from a json
type DICT ¶
type DICT struct { //LastUsed indicates when the token was used last LastUsed time.Time //Map has tokens mapped to their word Map map[string]Token }
DICT holds the dictionary in the system
type DICTAggregator ¶
DICTAggregator is the aggregator to get the dict from a service or database
type DICTRequest ¶
type DICTRequest struct { //ID to which the dictionary belong to ID string //Type is the type of the dictionary request. It can have Add, Get, Remove Type DICTRequestType //DICT is the dictionary under watch DICT DICT //Valid indicates that the dict is valid. During get requests, if valid is false then cache couldn't find the dict Valid bool //Out channel for sending response to the requester Out chan DICTRequest }
DICTRequest can be used to make a request to dictionary cache
type DICTRequestType ¶
type DICTRequestType uint
DICTRequestType is the type of the request for the dictionary
const ( //DICTAdd adds a dictionary for the given id DICTAdd DICTRequestType = 1 //DICTGet returns the dictionary of a given id DICTGet DICTRequestType = 2 //DICTRemove the dictionary from the cache DICTRemove DICTRequestType = 3 //DICTRemoveCheck will iterate over the dictionary and remove the unused dicts DICTRemoveCheck DICTRequestType = 4 //DICTPreCache will do a check whether dictionary is available for the user. If not will cache it DICTPreCache DICTRequestType = 5 //DICTUpdate will remove the dict from the cache and fetch the updated dict DICTUpdate DICTRequestType = 6 )
type FastToken ¶
type FastToken struct { //Pos is the position of the token in the root sentence Pos int //Word is the match word corresponding to the token Word []rune //Tables is the list of table nodes in the token Tables []TableNode //Columns is the list of column nodes in the token Columns []ColumnNode //Values is the list of value nodes in the token Values []ValueNode //Operators is the list of operators nodes in the token Operators []OperatorNode //Unknowns is the list of unknows nodes in the token Unknowns []UnknownNode //Times is the list of time nodes in the token Times []TimeNode }
FastToken is used to store the token with nodes converted into their concrete type so processing become becomes easy
type KMP ¶
type KMP struct { //Pos holds the position information about the pattern Pos []int //KMP Pattern Pattern []Type }
KMP holds the infomartaion on a pattern to find match with a given target array
type KnowledgeBaseNode ¶
type KnowledgeBaseNode struct { //UID is the unique id of the node UID string //Word is the word with which the knowledge base node has to be matched Word []rune //Name is the name of the node Name string //Children are the children node of the node Children []Node //Resolved indicates that the node is resolved Resolved bool //Description of the node Description string //KBType indicates the type of the knowledgebase KBType KnowledgeBaseType }
KnowledgeBaseNode can store info about any node. It doesn't have a parent node
func (*KnowledgeBaseNode) Copy ¶
func (k *KnowledgeBaseNode) Copy() Node
Copy will return a copy of the node
func (*KnowledgeBaseNode) ID ¶
func (k *KnowledgeBaseNode) ID() string
ID returns the unique id of the node
func (*KnowledgeBaseNode) IsResolved ¶
func (k *KnowledgeBaseNode) IsResolved() bool
IsResolved will return true if the node is resolved
func (*KnowledgeBaseNode) MarshalJSON ¶
func (k *KnowledgeBaseNode) MarshalJSON() ([]byte, error)
MarshalJSON encodes the node into a serializable json
func (*KnowledgeBaseNode) PID ¶
func (k *KnowledgeBaseNode) PID() string
PID returns the PUID if the node
func (*KnowledgeBaseNode) Parent ¶
func (k *KnowledgeBaseNode) Parent() Node
Parent returns the PN of the node
func (*KnowledgeBaseNode) SetResolved ¶
func (k *KnowledgeBaseNode) SetResolved(state bool)
SetResolved will set the resolved state of the node
func (*KnowledgeBaseNode) TokenWord ¶
func (k *KnowledgeBaseNode) TokenWord() []rune
TokenWord returns the word property of the node
func (*KnowledgeBaseNode) UnmarshalJSON ¶
func (k *KnowledgeBaseNode) UnmarshalJSON(data []byte) error
UnmarshalJSON decodes the node from a json
type KnowledgeBaseType ¶
type KnowledgeBaseType uint
KnowledgeBaseType indicates the type of the knowledgebase
const ( //SystemKB stands for the knowledgebase of the system SystemKB KnowledgeBaseType = 1 //UserKB stands for the knowledgebase of the user UserKB KnowledgeBaseType = 2 )
type Node ¶
type Node interface { //ID is the unique identifier of the node ID() string //Type is the type of the node Type() Type //TokenWord returns the word to be matched with the token TokenWord() []rune //PID returns the id of the parent associated with the node PID() string //Parent returns the parent node Parent() Node //MarshalJSON will encode a node to json string which can be stored MarshalJSON() ([]byte, error) //UnmarshalJSON will decode a json string to node UnmarshalJSON([]byte) error //IsResolved returns true if the node is resolved. IsResolved() bool //SetResolved will set the resolved state of the node SetResolved(bool) //Copy will make the copy of the node Copy() Node }
Node is the interface to be implemented for considering it as a basic building block in octopus
type OperatorNode ¶
type OperatorNode struct { //UID is the unique id of the operator node UID string //Word is the word with which the operator node has to be matched Word []rune //PUID is the UID of operator's parent node PUID string //PN is the parent node of the operator. It will be a KnowledgeBase PN Node //Resolved indicates that the node is resolved Resolved bool //Column is the column with which the operator is applied Column *ColumnNode //Unknown is the value to be applied to the column node with the operator Unknown *UnknownNode //Value is the value to be applied to the column node with the operator Value *ValueNode //Time is the time node to be applied to the column node with the operator Time *TimeNode //Operation is the operation applied by the node Operation string }
OperatorNode is the node storing the information about a operator. Filters are set based on this node. It depends upon a column and value/unknown
func (*OperatorNode) IsResolved ¶
func (o *OperatorNode) IsResolved() bool
IsResolved will return true if the node is resolved
func (*OperatorNode) MarshalJSON ¶
func (o *OperatorNode) MarshalJSON() ([]byte, error)
MarshalJSON encodes the node into a serializable json
func (*OperatorNode) Parent ¶
func (o *OperatorNode) Parent() Node
Parent returns the PN of the node
func (*OperatorNode) SetResolved ¶
func (o *OperatorNode) SetResolved(state bool)
SetResolved will set the resolved state of the node
func (*OperatorNode) TokenWord ¶
func (o *OperatorNode) TokenWord() []rune
TokenWord returns the word property of the node
func (*OperatorNode) UnmarshalJSON ¶
func (o *OperatorNode) UnmarshalJSON(data []byte) error
UnmarshalJSON decodes the node from a json
type Query ¶
type Query struct { //Tables has the map of tables whose data is being accessed by the query Tables map[string]TableNode `json:"tables,omitempty"` //Select has the list of columns to be selected from the data Select []ColumnNode `json:"select,omitempty"` //GroupBy has the list of columns to be used for group the data GroupBy []ColumnNode `json:"group_by,omitempty"` //Filters has the list of filters applied in the query Filters []OperatorNode `json:"filters,omitempty"` //Result has the result of the query Result []map[string]interface{} `json:"result,omitempty"` }
Query has the interpreted query info
func (Query) ToSingleTableSQL ¶
ToSingleTableSQL will convert the query to sql if the query has only one table
type Request ¶
type Request struct { //ID to which the tokenizer belong to ID string //Type is the type of the tokenizer request. It can have Add, Get, Remove Type RequestType //Tokenizer is the tokenizer under watch Tokenizer Tokenizer //Sentence is the sentence to be tokenized Sentence []rune //Valid indicates whethe the result is valid or not Valid bool //matches returns the matched tokens Matches []Token //Out channel for sending response to the requester Out chan Request }
Request can be used to make a request to tokenizer cache
type RequestType ¶
type RequestType uint
RequestType is the type of the request for the tokenizer
const ( //TokenizerAdd adds a tokenizer for the given id TokenizerAdd RequestType = 1 //TokenizerGet returns the tokenizer of a given id TokenizerGet RequestType = 2 //TokenizerRemove the tokenizer from the cache TokenizerRemove RequestType = 3 )
type Rule ¶
type Rule struct { //Name of the rule for debugging purposes Name string `json:"name,omitempty"` //Description of the rule. Description string `json:"description,omitempty"` //Disabled indicates wether the riules is emnabled Disabled bool `json:"disabled,omitempty"` //Template is the template of the of the rule Template []Type `json:"template,omitempty"` //Resolve function will try to run the resolution for the rule. //Query argument is the query to which the resolved tokens has to be attached //The int argument gives the index of the fasttokoen we are referring to //Resolve function should not mutate the state of the rule Resolve func(Query, []FastToken, int) (Query, error) `json:"-"` //Matches are the indices of the tokens in the list of tokens to which the rule template has found match Matches []int `json:"-"` //Pattern is the kmp buildup of the pattern Pattern *KMP `json:"-"` }
Rule represents a rule with template and resolver to resolve the parsed tokens
func MatchRules ¶
MatchRules will try to match the tokens passed with rules existing in the interpreter If no match is found, will return nil
type RuleGroup ¶
type RuleGroup struct { //Rules in the group Rules []Rule `json:"rules,omitempty"` //Tag identifier for the group Tag string `json:"tag,omitempty"` }
RuleGroup stores the list of rules to be executed together with priority
type SQLQuery ¶
type SQLQuery struct { //Query is the query string with arguments Query string //Args has the arguments to be passed to the query string Args []interface{} }
SQLQuery stores a sql query to be executed
type TableNode ¶
type TableNode struct { //UID is the unique id of the table node UID string //Word is the word with which the table node has to be matched Word []rune //PUID is the UID of table's parent node PUID string //PN is the parent node of the table. It will be a KnowledgeBase PN Node //Name is the name of the node Name string //Children are the children node of the table node Children []ColumnNode //Resolved indicates that the node is resolved Resolved bool //DefaultDateFieldUID is the uid of the default date field in the table DefaultDateFieldUID string //DefaultDateField is the default column to be selected as date in the table DefaultDateField *ColumnNode //Description of the node Description string //DatastoreID is Datastore to which the table belongs to DatastoreID uint }
TableNode is the node storing the information about a table. It can be the child of a KnowledgeBase and can have Column as children
func (*TableNode) IsResolved ¶
IsResolved will return true if the node is resolved
func (*TableNode) MarshalJSON ¶
MarshalJSON encodes the node into a serializable json
func (*TableNode) SetResolved ¶
SetResolved will set the resolved state of the node
func (*TableNode) UnmarshalJSON ¶
UnmarshalJSON decodes the node from a json
type TimeNode ¶
type TimeNode struct { //UID is the unique id of the time node UID string //Word is the word with which the time node has to be matched Word []rune //PUID is the UID of time's parent node PUID string //PN is the parent node of the time. It will be a KnowledgeBase PN Node //Value is the value of the timenode Value datetime.Value //Resolved indicates that the node is resolved Resolved bool }
TimeNode is the node storing the information about a time.
func (*TimeNode) IsResolved ¶
IsResolved will return true if the node is resolved
func (*TimeNode) MarshalJSON ¶
MarshalJSON encodes the node into a serializable json
func (*TimeNode) SetResolved ¶
SetResolved will set the resolved state of the node
func (*TimeNode) UnmarshalJSON ¶
UnmarshalJSON decodes the node from a json
type Token ¶
type Token struct { //Pos is the position of the token in the root sentence Pos int //Word is the match word corresponding to the token Word []rune //Node has the list of nodes applicable to a token Nodes []Node }
Token is the parsed word and possible nodes a word associated to
func AdjustPositions ¶
AdjustPositions will adjust the positions of the tokens corresponding to the position of the token in the token list rather than the sentence
func BuildTimeNodes ¶
BuildTimeNodes will insert time nodes if a valid response from date service is received for its place This function won't replace any existing tokens. If conflict between existing node and time node come, the time node will be skipped with priority given to the existing node.
func BuildUnknowns ¶
BuildUnknowns build unknown nodes. It return the tokens with unidentified words in the sentence transfomed to tokens with unknowns
type Tokenizer ¶
type Tokenizer struct { //Machine has the machine storing the trie Machine *goahocorasick.Machine //map has the tokens mapped to their word Map map[string]Token }
Tokenizer has the tokens map and machine for storing the state of the tokens trie
type Type ¶
type Type int
Type is type of node
const ( //KnowledgeBase is the collection of diffent tables etcs KnowledgeBase Type = 1 //Table is the table to which referring data belongs to Table Type = 2 //Column is the column that is being referred in a table Column Type = 3 //Value is the value that is present in the table's specific column Value Type = 4 //Operator is the operation to be applied when doing a filter Operator Type = 5 //GroupBy is the based on which the values of columns should be grouped GroupBy Type = 6 //AggregationFn is the aggregation function to be used for a column in a query AggregationFn Type = 7 //Unknown is a node whose purpose has still not been resolved Unknown Type = 8 //Ignore is node that has to be ignored without going for further processing Ignore Type = 9 //Context node if found indicates that there is an context to the query and certain values can be inferrerd from that Context Type = 10 //Time node represents a time data Time Type = 11 )
func BuildPattern ¶
BuildPattern will build pattern for the given tokens
type UnknownNode ¶
type UnknownNode struct { //UID is the unique id of the unkonown node UID string //Word is the word with which the unknown node has to be matched Word []rune //PUID is the UID of unknown's parent node PUID string //PN is the parent node of the unknown. It will be a KnowledgeBase PN Node //Resolved indicates that the node is resolved Resolved bool }
UnknownNode is the node storing the information about a unknown. Unkown tokens are tokens that are not identified by the system but can have potential information for resolving the query
func (*UnknownNode) IsResolved ¶
func (u *UnknownNode) IsResolved() bool
IsResolved will return true if the node is resolved
func (*UnknownNode) MarshalJSON ¶
func (u *UnknownNode) MarshalJSON() ([]byte, error)
MarshalJSON encodes the node into a serializable json
func (*UnknownNode) SetResolved ¶
func (u *UnknownNode) SetResolved(state bool)
SetResolved will set the resolved state of the node
func (*UnknownNode) TokenWord ¶
func (u *UnknownNode) TokenWord() []rune
TokenWord returns the word property of the node
func (*UnknownNode) UnmarshalJSON ¶
func (u *UnknownNode) UnmarshalJSON(data []byte) error
UnmarshalJSON decodes the node from a json
type ValueNode ¶
type ValueNode struct { //UID is the unique id of the value node UID string //Word is the word with which the value node has to be matched Word []rune //PN is the parent node of the value. It will be a column node PN *ColumnNode //PUID is the UID of value's parent node PUID string //Name is the name of the node Name string //Resolved indicates that the node is resolved Resolved bool }
ValueNode is the node storing the information about a value in a column. It can be the child of a Column Node and cannot have any children
func (*ValueNode) IsResolved ¶
IsResolved will return true if the node is resolved
func (*ValueNode) MarshalJSON ¶
MarshalJSON encodes the node into a serializable json
func (*ValueNode) SetResolved ¶
SetResolved will set the resolved state of the node
func (*ValueNode) UnmarshalJSON ¶
UnmarshalJSON decodes the node from a json