query

package
v0.0.1 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Nov 27, 2018 License: Apache-2.0 Imports: 23 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

View Source 
var BinaryExprTypeToCFunctorType = map[expr.Token]C.enum_BinaryFunctorType{
	expr.AND:         C.And,
	expr.OR:          C.Or,
	expr.EQ:          C.Equal,
	expr.NEQ:         C.NotEqual,
	expr.LT:          C.LessThan,
	expr.LTE:         C.LessThanOrEqual,
	expr.GT:          C.GreaterThan,
	expr.GTE:         C.GreaterThanOrEqual,
	expr.ADD:         C.Plus,
	expr.SUB:         C.Minus,
	expr.MUL:         C.Multiply,
	expr.DIV:         C.Divide,
	expr.MOD:         C.Mod,
	expr.BITWISE_AND: C.BitwiseAnd,
	expr.BITWISE_OR:  C.BitwiseOr,
	expr.BITWISE_XOR: C.BitwiseXor,
	expr.FLOOR:       C.Floor,
	expr.CONVERT_TZ:  C.Plus,
}

BinaryExprTypeToCFunctorType maps from binary operator to C BinaryFunctorType

View Source 
var ColumnTypeToExprType = map[string]expr.Type{
	metaCom.Bool:      expr.Boolean,
	metaCom.Int8:      expr.Signed,
	metaCom.Int16:     expr.Signed,
	metaCom.Int32:     expr.Signed,
	metaCom.Int64:     expr.Signed,
	metaCom.Uint8:     expr.Unsigned,
	metaCom.Uint16:    expr.Unsigned,
	metaCom.Uint32:    expr.Unsigned,
	metaCom.Float32:   expr.Float,
	metaCom.SmallEnum: expr.Unsigned,
	metaCom.BigEnum:   expr.Unsigned,
	metaCom.GeoPoint:  expr.GeoPoint,
	metaCom.GeoShape:  expr.GeoShape,
}

ColumnTypeToExprType maps data type from the column schema format to expression AST format.

View Source 
var DataTypeToCDataType = map[memCom.DataType]C.enum_DataType{
	memCom.Bool:      C.Bool,
	memCom.Int8:      C.Int8,
	memCom.Uint8:     C.Uint8,
	memCom.Int16:     C.Int16,
	memCom.Uint16:    C.Uint16,
	memCom.Int32:     C.Int32,
	memCom.Int64:     C.Int64,
	memCom.Uint32:    C.Uint32,
	memCom.Float32:   C.Float32,
	memCom.SmallEnum: C.Uint8,
	memCom.BigEnum:   C.Uint16,
	memCom.GeoPoint:  C.GeoPoint,
	memCom.UUID:      C.UUID,
}

DataTypeToCDataType mapps from memstore data type to c data types

View Source 
var UnaryExprTypeToCFunctorType = map[expr.Token]C.enum_UnaryFunctorType{
	expr.NOT:                 C.Not,
	expr.UNARY_MINUS:         C.Negate,
	expr.IS_NULL:             C.IsNull,
	expr.IS_NOT_NULL:         C.IsNotNull,
	expr.BITWISE_NOT:         C.BitwiseNot,
	expr.GET_WEEK_START:      C.GetWeekStart,
	expr.GET_MONTH_START:     C.GetMonthStart,
	expr.GET_QUARTER_START:   C.GetQuarterStart,
	expr.GET_YEAR_START:      C.GetYearStart,
	expr.GET_DAY_OF_MONTH:    C.GetDayOfMonth,
	expr.GET_DAY_OF_YEAR:     C.GetDayOfYear,
	expr.GET_MONTH_OF_YEAR:   C.GetMonthOfYear,
	expr.GET_QUARTER_OF_YEAR: C.GetQuarterOfYear,
	expr.GET_HLL_VALUE:       C.GetHLLValue,
}

UnaryExprTypeToCFunctorType maps from unary operator to C UnaryFunctorType

Functions

func GetCurrentCalendarUnit 

func GetCurrentCalendarUnit(base time.Time, unit string) (start, end time.Time, err error)

GetCurrentCalendarUnit returns the start and end of the calendar unit for base.

Types

type AQLQuery 

type AQLQuery struct {
	// Name of the main table.
	Table string `json:"table"`

	// Foreign tables to be joined.
	Joins []Join `json:"joins,omitempty"`

	// Dimensions to group by on.
	Dimensions []Dimension `json:"dimensions,omitempty"`

	// Measures/metrics to report.
	Measures []Measure `json:"measures"`

	// Row level filters to apply for all measures. The filters are ANDed togther.
	Filters []string `json:"rowFilters,omitempty"`

	// Syntax sugar for specifying a time based range filter.
	TimeFilter TimeFilter `json:"timeFilter,omitempty"`

	// Timezone to use when converting timestamp to calendar time, specified as:
	//   - -8:00
	//   - GMT
	//   - America/Los_Angeles
	//   - timezone(city_id)
	//   - region_timezone(city_id)
	//   - mega_region_timezone(city_id)
	//   - sub_region_timezone(city_id)
	//   - country_timezone(city_id)
	Timezone string `json:"timezone,omitempty"`

	// This overrides "now" (in seconds)
	Now int64 `json:"now,omitempty"`
	// contains filtered or unexported fields
}

AQLQuery specifies the query on top of tables.

func (*AQLQuery) Compile 

func (q *AQLQuery) Compile(store memstore.MemStore, returnHLL bool) *AQLQueryContext

Compile returns the compiled AQLQueryContext for data feeding and query execution. Caller should check for AQLQueryContext.Error.

type AQLQueryContext 

type AQLQueryContext struct {
	// The query input.
	Query *AQLQuery `json:"query"`

	// Context for one-operator-per-kernel execution.
	OOPK OOPKContext `json:"oopk"`

	// Scanner for all tables. [0] for the main table; [1:] for tables in joins.
	TableScanners []*TableScanner `json:"scanners"`
	// Map from table alias to ID (index to TableScanners).
	TableIDByAlias map[string]int `json:"tableIDs"`
	// Map from table name to schema for convenience. In case of self join,
	// only one entry is referenced here by the name of the table.
	TableSchemaByName map[string]*memstore.TableSchema `json:"-"`
	// Index to filters in Query.Filters that are identified as prefilters.
	Prefilters []int `json:"prefilters,omitempty"`

	Error error `json:"error,omitempty"`

	Device int `json:"device"`

	Debug bool `json:"debug,omitempty"`

	Profiling string `json:"profiling,omitempty"`

	Results queryCom.AQLTimeSeriesResult `json:"-"`

	// whether to serialize the query result as HLLData. If ReturnHLLData is true, we will not release dimension
	// vector and measure vector until serialization is done.
	ReturnHLLData  bool   `json:"ReturnHLLData"`
	HLLQueryResult []byte `json:"-"`
	// contains filtered or unexported fields
}

AQLQueryContext stores all contextual data for handling an AQL query.

func (*AQLQueryContext) FindDeviceForQuery 

func (qc *AQLQueryContext) FindDeviceForQuery(memStore memstore.MemStore, preferredDevice int,
	deviceManager *DeviceManager, timeout int)

FindDeviceForQuery calls device manager to find a device for the query

func (*AQLQueryContext) Postprocess 

func (qc *AQLQueryContext) Postprocess() queryCom.AQLTimeSeriesResult

Postprocess converts the internal dimension and measure vector in binary format to AQLTimeSeriesResult nested result format. It also translates enum values back to their string representations.

func (*AQLQueryContext) PostprocessAsHLLData 

func (qc *AQLQueryContext) PostprocessAsHLLData() ([]byte, error)

PostprocessAsHLLData serializes the query result into HLLData format. It will also release the device memory after serialization.

func (*AQLQueryContext) ProcessQuery 

func (qc *AQLQueryContext) ProcessQuery(memStore memstore.MemStore)

ProcessQuery processes the compiled query and executes it on GPU.

func (*AQLQueryContext) Release 

func (qc *AQLQueryContext) Release()

Release releases all device memory it allocated. It **should only called** when any errors happens while the query is processed.

func (*AQLQueryContext) ReleaseHostResultsBuffers 

func (qc *AQLQueryContext) ReleaseHostResultsBuffers()

ReleaseHostResultsBuffers deletes the result buffer from host memory after postprocessing

func (*AQLQueryContext) Rewrite 

func (qc *AQLQueryContext) Rewrite(expression expr.Expr) expr.Expr

Rewrite walks the expresison AST and resolves data types bottom up. In addition it also translates enum strings and rewrites their predicates.

func (*AQLQueryContext) SerializeHLL 

func (qc *AQLQueryContext) SerializeHLL(dataTypes []memCom.DataType,
	enumDicts map[int][]string, timeDimensions []int) ([]byte, error)

SerializeHLL allocates buffer based on the metadata and then serializes hll data into the buffer.

type AQLRequest 

type AQLRequest struct {
	Queries []AQLQuery `json:"queries"`
}

AQLRequest contains multiple of AQLQueries.

type AQLResponse 

type AQLResponse struct {
	Results      []queryCom.AQLTimeSeriesResult `json:"results"`
	Errors       []error                        `json:"errors,omitempty"`
	QueryContext []*AQLQueryContext             `json:"context,omitempty"`
}

AQLResponse contains results for multiple AQLQueries.

type DeviceInfo 

type DeviceInfo struct {
	// device id
	DeviceID int `json:"deviceID"`
	// number of queries being served by device
	QueryCount int `json:"queryCount"`
	// device capacity.
	TotalMemory int `json:"totalMemory"`
	// device available capacity.
	TotalAvailableMemory int `json:"totalAvailableMemory"`
	// total free memory
	FreeMemory int `json:"totalFreeMemory"`
	// query to memory map
	QueryMemoryUsageMap map[*AQLQuery]int `json:"-"`
}

DeviceInfo stores memory information per device

type DeviceManager 

type DeviceManager struct {
	// lock to sync ops.
	sync.RWMutex `json:"-"`
	// device to DeviceInfo map
	DeviceInfos []*DeviceInfo `json:"deviceInfos"`
	// default DeviceChoosingTimeout for finding a device
	Timeout int `json:"timeout"`
	// Max available memory, this can be used to early determined whether a query can be satisfied or not.
	MaxAvailableMemory int `json:"maxAvailableMemory"`
	// contains filtered or unexported fields
}

DeviceManager has the following functionalities: 1. Keep track of number of queries being served by this device and memory usage info 2. Estimate the memory requirement for a given query and determine if a device has enough memory to process a query 3. Assign queries to chosen device according to routing strategy specified

func NewDeviceManager 

func NewDeviceManager(cfg common.QueryConfig) *DeviceManager

NewDeviceManager is used to init a DeviceManager.

func (*DeviceManager) FindDevice 

func (d *DeviceManager) FindDevice(query *AQLQuery, requiredMem int, preferredDevice int, timeout int) int

FindDevice finds a device to run a given query. If a device is not found, it will wait until the DeviceChoosingTimeout seconds elapse.

func (*DeviceManager) ReleaseReservedMemory 

func (d *DeviceManager) ReleaseReservedMemory(device int, query *AQLQuery)

ReleaseReservedMemory adjust total free global memory for a given device after a query is complete

type Dimension 

type Dimension struct {
	// The SQL expression for computing the dimension.
	// Expr can be empty when TimeBucketizer is specified, which implies the
	// designated time column from the main table is used as the expresssion.
	Expr string `json:"sqlExpression"`

	// Decides how to bucketize a timestamp Dimension before grouping by.
	// See https://github.com/uber/aresdb/wiki/aql#time_bucketizer
	TimeBucketizer string `json:"timeBucketizer,omitempty"`

	TimeUnit string `json:"timeUnit,omitempty"`

	// Bucketizes numeric dimensions for integers and floating point numbers.
	NumericBucketizer NumericBucketizerDef `json:"numericBucketizer,omitempty"`
	// contains filtered or unexported fields
}

Dimension specifies a row level dimension for grouping by.

type HLLDataWriter 

type HLLDataWriter struct {
	queryCom.HLLData
	// contains filtered or unexported fields
}

HLLDataWriter is the struct to serialize HLL Data struct.

func (*HLLDataWriter) SerializeHeader 

func (builder *HLLDataWriter) SerializeHeader() error

SerializeHeader first serializes header into buffer the builder holds.

type HLLQueryResults 

type HLLQueryResults struct {
	// contains filtered or unexported fields
}

HLLQueryResults holds the buffer to store multiple hll query results or errors.

func NewHLLQueryResults 

func NewHLLQueryResults() *HLLQueryResults

NewHLLQueryResults returns a new NewHLLQueryResults and writes the magical header and padding to underlying buffer.

func (*HLLQueryResults) GetBytes 

func (r *HLLQueryResults) GetBytes() []byte

GetBytes returns the underlying bytes.

func (*HLLQueryResults) WriteError 

func (r *HLLQueryResults) WriteError(err error)

WriteError write error to the buffer.

func (*HLLQueryResults) WriteResult 

func (r *HLLQueryResults) WriteResult(result []byte)

WriteResult write result to the buffer.

type Join 

type Join struct {
	// Name of the table to join against.
	Table string `json:"table"`

	// Alias for the table. Empty means the table name will be used as alias.
	Alias string `json:"alias"`

	// Condition expressions to be ANDed together for the join.
	Conditions []string `json:"conditions"`
	// contains filtered or unexported fields
}

Join specifies a secondary table to be explicitly joined in the query.

type Measure 

type Measure struct {
	// The SQL expression for computing the measure.
	Expr string `json:"sqlExpression"`

	// Row level filters to apply for this measure.
	// The filters are ANDed togther.
	Filters []string `json:"rowFilters,omitempty"`
	// contains filtered or unexported fields
}

Measure specifies a group level aggregation measure.

type NumericBucketizerDef 

type NumericBucketizerDef struct {

	// Generates equal-width buckets. BucketWidth should be positive.
	// The generated buckets are:
	// ... [-2w, -w), [-w, 0), [0, w), [w, 2w) ...
	BucketWidth float64 `json:"bucketWidth,omitempty"`

	// Generates exponential/log buckets. LogBase should be positive.
	// The generated buckets are:
	// ... [pow(b, -2), pow(b, -1)), [pow(b, -1), 1), [1, pow(b, 1)), [pow(b, 1), pow(b, 2)) ...
	LogBase float64 `json:"logBase,omitempty"`

	// Generates a fixed number of buckets using the specified partitions.
	// The numbers should be in sorted order. The generated buckets are:
	// [-inf, p0), [p0, p1), [p1, p2), ... [pn-1, inf)
	ManualPartitions []float64 `json:"manualPartitions,omitempty"`
}

NumericBucketizerDef defines how numbers should be bucketized before being grouped by as a dimension. The returned dimension is a string in the format of `lower_bound`, representing `[lower_bound, uper_bound)`.

type OOPKContext 

type OOPKContext struct {

	// Filters that apply to all archive and live batches.
	// MainTableCommonFilters match filters with only main table columns involved
	MainTableCommonFilters []expr.Expr `json:"mainTableCommonFilters,omitempty"`
	// ForeignTableCommonFilters match filters with foreign table columns involved
	ForeignTableCommonFilters []expr.Expr `json:"foreignTableCommonFilters,omitempty"`
	// Lower bound [0] and upper bound [1] time filter. nil if not applicable.
	// [0] should be applied to the first archive batch and all live batches.
	// [1] should be applied to the last archive batch and all live batches.
	TimeFilters [2]expr.Expr `json:"timeFilters"`
	// Prefilters that only apply to live batches.
	// Archiving cutoff filtering is processed directly by the query engine and not
	// included here (different shards may have different cutoffs).
	Prefilters []expr.Expr `json:"prefilters,omitempty"`

	// Compiled and annotated ASTs for dimensions and measure.
	Dimensions []expr.Expr `json:"dimensions"`
	// Index of single dimension vector in global dimension vector
	// Following sorted order based on bytes
	DimensionVectorIndex []int `json:"dimensionVectorIndex"`
	// Number of dimensions per dim width
	NumDimsPerDimWidth queryCom.DimCountsPerDimWidth `json:"numDims"`
	// Dim row bytes is the sum number of bytes of all dimension values
	// plus validity bytes, for memory allocation convenience
	DimRowBytes int `json:"dimRowBytes"`

	// For one-operator-per-kernel we only support one measure per query.
	Measure       expr.Expr                `json:"measure"`
	MeasureBytes  int                      `json:"measureBytes"`
	AggregateType C.enum_AggregateFunction `json:"aggregate"`

	ResultSize int `json:"resultSize"`

	// For reporting purpose only.
	DeviceMemoryRequirement int           `json:"deviceMem"`
	DurationWaitedForDevice time.Duration `json:"durationWaitedForDevice"`

	// Stores the overall query stats for live batches and archive batches.
	LiveBatchStats    oopkQueryStats `json:"liveStats"`
	ArchiveBatchStats oopkQueryStats `json:"archiveStats"`
	// contains filtered or unexported fields
}

OOPKContext defines additional query context for one-operator-per-kernel execution.

type TableScanner 

type TableScanner struct {
	// Snapshot of the table schema for convenience.
	Schema *memstore.TableSchema `json:"-"`
	// IDS of all table shards to be scanned on this instance.
	Shards []int `json:"shards"`
	// IDs of columns to be used in this query, in the following order:
	//   1. Columns not from ArchivingSortColumns.
	//   2. Columns from ArchivingSortColumns in reverse order.
	Columns []int `json:"columns"`
	// reversed mapping from columnID to column scan order index
	ColumnsByIDs map[int]int `json:"-"`

	// Map from column ID to its usage by the query.
	ColumnUsages map[int]columnUsage `json:"columnUsage"`

	// Values of equality prefilters in order. Each 4 bytes of the uint32 is used
	// to store any data type other than UUID (not supported).
	EqualityPrefilterValues []uint32 `json:"equalityPrefilterValues,omitempty"`
	// Boundary types and values of the final range prefilter.
	RangePrefilterBoundaries [2]boundaryType `json:"rangePrefilterBoundaries"`
	RangePrefilterValues     [2]uint32       `json:"rangePrefilterValues"`
	// Range of archive batches to process: [Start, end).
	// Depending on the archiving progress of each shard, live batches may be
	// skipped for processing if the archiving cutoff is after the time of
	// ArchiveBatchIDEnd.
	ArchiveBatchIDStart int `json:"archiveBatchIDStart"`
	ArchiveBatchIDEnd   int `json:"archiveBatchIDEnd"`
}

TableScanner defines how data for a table should be fed to device memory for processing (scanner in a traditional terminology).

type TimeFilter 

type TimeFilter struct {
	// A table time column in the format of column, or table_alias.column.
	// When empty, it defaults to the designated time column of the main table.
	Column string `json:"column"`

	// The time specified in from and to are both inclusive.
	// See https://github.com/uber/aresdb/wiki/aql#time_filter
	From string `json:"from"`
	To   string `json:"to"`
}

TimeFilter is a syntax sugar for specifying time range.

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.