query

package
v0.0.2 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Sep 5, 2019 License: Apache-2.0 Imports: 23 Imported by: 3

Documentation

Index

Constants

This section is empty.

Variables

View Source 
var BinaryExprTypeToCFunctorType = map[expr.Token]C.enum_BinaryFunctorType{
	expr.AND:         C.And,
	expr.OR:          C.Or,
	expr.EQ:          C.Equal,
	expr.NEQ:         C.NotEqual,
	expr.LT:          C.LessThan,
	expr.LTE:         C.LessThanOrEqual,
	expr.GT:          C.GreaterThan,
	expr.GTE:         C.GreaterThanOrEqual,
	expr.ADD:         C.Plus,
	expr.SUB:         C.Minus,
	expr.MUL:         C.Multiply,
	expr.DIV:         C.Divide,
	expr.MOD:         C.Mod,
	expr.BITWISE_AND: C.BitwiseAnd,
	expr.BITWISE_OR:  C.BitwiseOr,
	expr.BITWISE_XOR: C.BitwiseXor,
	expr.FLOOR:       C.Floor,
	expr.CONVERT_TZ:  C.Plus,
}

BinaryExprTypeToCFunctorType maps from binary operator to C BinaryFunctorType

View Source 
var DataTypeToCDataType = map[memCom.DataType]C.enum_DataType{
	memCom.Bool:      C.Bool,
	memCom.Int8:      C.Int8,
	memCom.Uint8:     C.Uint8,
	memCom.Int16:     C.Int16,
	memCom.Uint16:    C.Uint16,
	memCom.Int32:     C.Int32,
	memCom.Int64:     C.Int64,
	memCom.Uint32:    C.Uint32,
	memCom.Float32:   C.Float32,
	memCom.SmallEnum: C.Uint8,
	memCom.BigEnum:   C.Uint16,
	memCom.GeoPoint:  C.GeoPoint,
	memCom.UUID:      C.UUID,
}

DataTypeToCDataType mapps from memstore data type to c data types

View Source 
var UnaryExprTypeToCFunctorType = map[expr.Token]C.enum_UnaryFunctorType{
	expr.NOT:                 C.Not,
	expr.UNARY_MINUS:         C.Negate,
	expr.IS_NULL:             C.IsNull,
	expr.IS_NOT_NULL:         C.IsNotNull,
	expr.BITWISE_NOT:         C.BitwiseNot,
	expr.GET_WEEK_START:      C.GetWeekStart,
	expr.GET_MONTH_START:     C.GetMonthStart,
	expr.GET_QUARTER_START:   C.GetQuarterStart,
	expr.GET_YEAR_START:      C.GetYearStart,
	expr.GET_DAY_OF_MONTH:    C.GetDayOfMonth,
	expr.GET_DAY_OF_YEAR:     C.GetDayOfYear,
	expr.GET_MONTH_OF_YEAR:   C.GetMonthOfYear,
	expr.GET_QUARTER_OF_YEAR: C.GetQuarterOfYear,
	expr.GET_HLL_VALUE:       C.GetHLLValue,
}

UnaryExprTypeToCFunctorType maps from unary operator to C UnaryFunctorType

Functions

This section is empty.

Types

type AQLQueryContext 

type AQLQueryContext struct {
	// The query input.
	Query *queryCom.AQLQuery `json:"query"`

	// Context for one-operator-per-kernel execution.
	OOPK OOPKContext `json:"oopk"`

	// Scanner for all tables. [0] for the main table; [1:] for tables in joins.
	TableScanners []*TableScanner `json:"scanners"`
	// Map from table alias to ID (index to TableScanners).
	TableIDByAlias map[string]int `json:"tableIDs"`
	// Map from table name to schema for convenience. In case of self join,
	// only one entry is referenced here by the name of the table.
	TableSchemaByName map[string]*memCom.TableSchema `json:"-"`
	// Index to filters in Query.Filters that are identified as prefilters.
	Prefilters []int `json:"prefilters,omitempty"`

	Error error `json:"error,omitempty"`

	Device int `json:"device"`

	Debug bool `json:"debug,omitempty"`

	Profiling string `json:"profiling,omitempty"`

	Results queryCom.AQLQueryResult `json:"-"`

	// whether it's a DataOnly request from broker
	DataOnly bool `json:"DataOnly"`
	// whether to serialize the query result as HLLData. If ReturnHLLData is true, we will not release dimension
	// vector and measure vector until serialization is done.
	ReturnHLLData  bool   `json:"ReturnHLLData"`
	HLLQueryResult []byte `json:"-"`

	// fields for non aggregate query
	// Flag to indicate if this query is not aggregation query
	IsNonAggregationQuery bool

	// for eager flush query result
	ResponseWriter http.ResponseWriter
	// contains filtered or unexported fields
}

AQLQueryContext stores all contextual data for handling an AQL query.

func (*AQLQueryContext) Compile added in v0.0.2 

func (qc *AQLQueryContext) Compile(tableSchemaReader memCom.TableSchemaReader, shardOwner topology.ShardOwner)

Compile compiles AQLQueryContext for data feeding and query execution. Caller should check for AQLQueryContext.Error.

func (*AQLQueryContext) FindDeviceForQuery 

func (qc *AQLQueryContext) FindDeviceForQuery(memStore memstore.MemStore, preferredDevice int,
	deviceManager *DeviceManager, timeout int)

FindDeviceForQuery calls device manager to find a device for the query

func (*AQLQueryContext) Postprocess 

func (qc *AQLQueryContext) Postprocess()

Postprocess converts the internal dimension and measure vector in binary format to AQLQueryResult nested result format. It also translates enum values back to their string representations.

func (*AQLQueryContext) PostprocessAsHLLData 

func (qc *AQLQueryContext) PostprocessAsHLLData() ([]byte, error)

PostprocessAsHLLData serializes the query result into HLLData format. It will also release the device memory after serialization.

func (*AQLQueryContext) ProcessQuery 

func (qc *AQLQueryContext) ProcessQuery(memStore memstore.MemStore)

ProcessQuery processes the compiled query and executes it on GPU.

func (*AQLQueryContext) Release 

func (qc *AQLQueryContext) Release()

Release releases all device memory it allocated. It **should only called** when any errors happens while the query is processed.

func (*AQLQueryContext) ReleaseHostResultsBuffers 

func (qc *AQLQueryContext) ReleaseHostResultsBuffers()

ReleaseHostResultsBuffers deletes the result buffer from host memory after postprocessing

func (*AQLQueryContext) ResultsRowsFlushed added in v0.0.2 

func (qc *AQLQueryContext) ResultsRowsFlushed() int

func (*AQLQueryContext) Rewrite 

func (qc *AQLQueryContext) Rewrite(expression expr.Expr) expr.Expr

Rewrite walks the expresison AST and resolves data types bottom up. In addition it also translates enum strings and rewrites their predicates.

func (*AQLQueryContext) SerializeHLL 

func (qc *AQLQueryContext) SerializeHLL(dataTypes []memCom.DataType,
	enumDicts map[int][]string, timeDimensions []int) ([]byte, error)

SerializeHLL allocates buffer based on the metadata and then serializes hll data into the buffer.

type BatchExecutor added in v0.0.2 

type BatchExecutor interface {
	// contains filtered or unexported methods
}

BatchExecutor is batch executor interface for both Non-aggregation query and Aggregation query

func NewBatchExecutor added in v0.0.2 

func NewBatchExecutor(qc *AQLQueryContext, batchID int32, customFilterFunc customFilterExecutor, stream unsafe.Pointer, start time.Time) BatchExecutor

NewBatchExecutor is to create a BatchExecutor.

func NewDummyBatchExecutor added in v0.0.2 

func NewDummyBatchExecutor() BatchExecutor

NewDummyBatchExecutor create a dummy BatchExecutor

type BatchExecutorImpl added in v0.0.2 

type BatchExecutorImpl struct {
	// contains filtered or unexported fields
}

BatchExecutorImpl is batch executor implementation for original aggregation query

type DeviceInfo 

type DeviceInfo struct {
	// device id
	DeviceID int `json:"deviceID"`
	// number of queries being served by device
	QueryCount int `json:"queryCount"`
	// device capacity.
	TotalMemory int `json:"totalMemory"`
	// device available capacity.
	TotalAvailableMemory int `json:"totalAvailableMemory"`
	// total free memory
	FreeMemory int `json:"totalFreeMemory"`
	// query to memory map
	QueryMemoryUsageMap map[*queryCom.AQLQuery]int `json:"-"`
}

DeviceInfo stores memory information per device

type DeviceManager 

type DeviceManager struct {
	// lock to sync ops.
	*sync.RWMutex `json:"-"`
	// device to DeviceInfo map
	DeviceInfos []*DeviceInfo `json:"deviceInfos"`
	// default DeviceChoosingTimeout for finding a device
	Timeout int `json:"timeout"`
	// Max available memory, this can be used to early determined whether a query can be satisfied or not.
	MaxAvailableMemory int `json:"maxAvailableMemory"`
	// contains filtered or unexported fields
}

DeviceManager has the following functionalities: 1. Keep track of number of queries being served by this device and memory usage info 2. Estimate the memory requirement for a given query and determine if a device has enough memory to process a query 3. Assign queries to chosen device according to routing strategy specified

func NewDeviceManager 

func NewDeviceManager(cfg common.QueryConfig) *DeviceManager

NewDeviceManager is used to init a DeviceManager.

func (*DeviceManager) FindDevice 

func (d *DeviceManager) FindDevice(query *queryCom.AQLQuery, requiredMem int, preferredDevice int, timeout int) int

FindDevice finds a device to run a given query. If a device is not found, it will wait until the DeviceChoosingTimeout seconds elapse.

func (*DeviceManager) ReleaseReservedMemory 

func (d *DeviceManager) ReleaseReservedMemory(device int, query *queryCom.AQLQuery)

ReleaseReservedMemory adjust total free global memory for a given device after a query is complete

type DummyBatchExecutorImpl added in v0.0.2 

type DummyBatchExecutorImpl struct {
}

DummyBatchExecutorImpl is a dummy executor which do nothing

type NonAggrBatchExecutorImpl added in v0.0.2 

type NonAggrBatchExecutorImpl struct {
	*BatchExecutorImpl
}

NonAggrBatchExecutorImpl is batch executor implementation for non-aggregation query

type OOPKContext 

type OOPKContext struct {

	// Filters that apply to all archive and live batches.
	// MainTableCommonFilters match filters with only main table columns involved
	MainTableCommonFilters []expr.Expr `json:"mainTableCommonFilters,omitempty"`
	// ForeignTableCommonFilters match filters with foreign table columns involved
	ForeignTableCommonFilters []expr.Expr `json:"foreignTableCommonFilters,omitempty"`
	// Lower bound [0] and upper bound [1] time filter. nil if not applicable.
	// [0] should be applied to the first archive batch and all live batches.
	// [1] should be applied to the last archive batch and all live batches.
	TimeFilters [2]expr.Expr `json:"timeFilters"`
	// Prefilters that only apply to live batches.
	// Archiving cutoff filtering is processed directly by the query engine and not
	// included here (different shards may have different cutoffs).
	Prefilters []expr.Expr `json:"prefilters,omitempty"`

	// Compiled and annotated ASTs for dimensions and measure.
	Dimensions []expr.Expr `json:"dimensions"`
	// Index of single dimension vector in global dimension vector
	// Following sorted order based on bytes
	DimensionVectorIndex []int `json:"dimensionVectorIndex"`
	// Number of dimensions per dim width
	NumDimsPerDimWidth queryCom.DimCountsPerDimWidth `json:"numDims"`
	// Dim row bytes is the sum number of bytes of all dimension values
	// plus validity bytes, for memory allocation convenience
	DimRowBytes int `json:"dimRowBytes"`

	// For one-operator-per-kernel we only support one measure per query.
	Measure       expr.Expr                `json:"measure"`
	MeasureBytes  int                      `json:"measureBytes"`
	AggregateType C.enum_AggregateFunction `json:"aggregate"`

	ResultSize int `json:"resultSize"`

	// For reporting purpose only.
	DeviceMemoryRequirement int           `json:"deviceMem"`
	DurationWaitedForDevice time.Duration `json:"durationWaitedForDevice"`

	// Stores the overall query stats for live batches and archive batches.
	LiveBatchStats    oopkQueryStats `json:"liveStats"`
	ArchiveBatchStats oopkQueryStats `json:"archiveStats"`
	// contains filtered or unexported fields
}

OOPKContext defines additional query context for one-operator-per-kernel execution.

func (*OOPKContext) IsHLL added in v0.0.2 

func (ctx *OOPKContext) IsHLL() bool

IsHLL return if the aggregation function is HLL

func (*OOPKContext) UseHashReduction added in v0.0.2 

func (ctx *OOPKContext) UseHashReduction() bool

UseHashReduction return whether to use hash reduction or not

type TableScanner 

type TableScanner struct {
	// Snapshot of the table schema for convenience.
	Schema *memCom.TableSchema `json:"-"`
	// IDS of all table shards to be scanned on this instance.
	Shards []int `json:"shards"`
	// IDs of columns to be used in this query, in the following order:
	//   1. Columns not from ArchivingSortColumns.
	//   2. Columns from ArchivingSortColumns in reverse order.
	Columns []int `json:"columns"`
	// reversed mapping from columnID to column scan order index
	ColumnsByIDs map[int]int `json:"-"`

	// Map from column ID to its usage by the query.
	ColumnUsages map[int]columnUsage `json:"columnUsage"`

	// Values of equality prefilters in order. Each 4 bytes of the uint32 is used
	// to store any data type other than UUID (not supported).
	EqualityPrefilterValues []uint32 `json:"equalityPrefilterValues,omitempty"`
	// Boundary types and values of the final range prefilter.
	RangePrefilterBoundaries [2]boundaryType `json:"rangePrefilterBoundaries"`
	RangePrefilterValues     [2]uint32       `json:"rangePrefilterValues"`
	// Range of archive batches to process: [Start, end).
	// Depending on the archiving progress of each shard, live batches may be
	// skipped for processing if the archiving cutoff is after the time of
	// ArchiveBatchIDEnd.
	ArchiveBatchIDStart int `json:"archiveBatchIDStart"`
	ArchiveBatchIDEnd   int `json:"archiveBatchIDEnd"`
}

TableScanner defines how data for a table should be fed to device memory for processing (scanner in a traditional terminology).

Source Files

View all Source files

Directories

Path Synopsis
sql
antlrgen
tree
util

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.