Documentation ¶
Index ¶
- Constants
- func AsString(value bigquery.Value) string
- func AsTime(value bigquery.Value) (time.Time, error)
- func DeleteGSObject(ctx context.Context, URL string) error
- func DownloadGSContent(ctx context.Context, URL string) ([]byte, error)
- func ExistsGSObject(ctx context.Context, URL string) bool
- func GetTableMeta(ctx context.Context, project, dataset, table string) (*bigquery.TableMetadata, error)
- func Handle(w http.ResponseWriter, r *http.Request)
- func HandleRequest(w http.ResponseWriter, r *http.Request)
- func IsNotFoundError(err error) bool
- func RunBQQuery(ctx context.Context, project, datasetLocation string, SQL string, ...) error
- func UploadGSContent(ctx context.Context, URL string, reader io.Reader) error
- type FormFields
- type Meta
- type NotFoundError
- type Request
- type Response
- type Service
- type TableInfo
- func GetTablesInfo(ctx context.Context, request *Request, lastModified bool) ([]*TableInfo, error)
- func GetTablesInfoFromSQL(ctx context.Context, projectID, datasetLocation, SQL string, useLegacy bool) ([]*TableInfo, error)
- func NewTableInfo(projectId, datasetID string, tableID string, created time.Time, ...) *TableInfo
- type TimeWindow
- type WindowedTable
Constants ¶
const ( TableInfoStandardSQL = "SELECT table_schema AS datasetId, table_name As tableId, UNIX_MILLIS(creation_time) AS created, UNIX_MILLIS(storage_last_modified_time) AS lastModified " + "FROM `%s.INFORMATION_SCHEMA.TABLE_STORAGE` " + "WHERE UNIX_MILLIS(storage_last_modified_time) > %v " + "AND table_schema = '%s' " + "ORDER BY storage_last_modified_time DESC" LastModifiedTableStandardSQL = "SELECT table_schema AS datasetId, table_name As tableId, UNIX_MILLIS(creation_time) AS created, UNIX_MILLIS(storage_last_modified_time) AS lastModified " + "FROM `%s.INFORMATION_SCHEMA.TABLE_STORAGE` " + "WHERE table_schema = '%s' " + "ORDER BY storage_last_modified_time DESC LIMIT 1" TableInfoLegacySQL = `` /* 212-byte string literal not displayed */ LastModifiedTableLegacySQL = `` /* 186-byte string literal not displayed */ )
Variables ¶
This section is empty.
Functions ¶
func DeleteGSObject ¶
DeleteGSObject delete gs object
func DownloadGSContent ¶
DownloadGSContent returns google storage content
func ExistsGSObject ¶
ExistsGSObject returns true if gs object exists
func GetTableMeta ¶
func Handle ¶
func Handle(w http.ResponseWriter, r *http.Request)
Handle windowed tables for supplied datasets, and meta file URL
func HandleRequest ¶
func HandleRequest(w http.ResponseWriter, r *http.Request)
func IsNotFoundError ¶
IsNotFoundError checks is supplied error is NotFoundError type
Types ¶
type FormFields ¶
type FormFields struct { Meta string Method string Dataset string Mode string Match string Location string Expr string Loopback string Prune string AbsoluteExpr string StorageRegion string }
func NewFormFields ¶
func NewFormFields(request *http.Request) (*FormFields, error)
func (*FormFields) AsRequest ¶
func (f *FormFields) AsRequest() (*Request, error)
func (*FormFields) Validate ¶
func (f *FormFields) Validate() error
type Meta ¶
type Meta struct { URL string DatasetID string Tables []*WindowedTable Expression string `description:"represents recently changed tables ranged decorator relative expression (without project id)"` AbsoluteExpression string `description:"represents recently changed tables ranged decorator absolute expression (with project id)"` // contains filtered or unexported fields }
func (*Meta) Match ¶
func (m *Meta) Match(matchExpressions []string) []*WindowedTable
func (*Meta) Prune ¶
Prune removes windowed table info that have not been update since: now - threshold
func (*Meta) ResetChangeFlag ¶
func (m *Meta) ResetChangeFlag()
func (*Meta) SortLastModifiedDesc ¶
func (m *Meta) SortLastModifiedDesc()
type NotFoundError ¶
type NotFoundError struct {
URL string
}
NotFoundError represents not found error
func (*NotFoundError) Error ¶
func (e *NotFoundError) Error() string
type Request ¶
type Request struct { Mode string `description:"operation mode: r - take snapshot, w - persist snapshot"` MetaURL string `description:"meta-file location, if relative path is used it adds gs:// protocol"` Location string `description:"dataset location"` DatasetID string `description:"source dataset"` MatchingTables []string `description:"matching table contain expression"` PruneThresholdInSec int `description:"max allowed duration in sec for unchanged windowed tables before removing"` LoopbackWindowInSec int `description:"dataset max loopback window for checking changed tables in supplied dataset"` Expression bool `description:"if expression flag is set it returns only relative expression (without poejct id)"` AbsoluteExpression bool `description:"if expression flag is set it returns only abslute expression (with poejct id)"` Method string `description:"data insert method: stream or load by default"` StorageRegion string `description:"storageRegion for standard sql"` }
Request represents a request for windowed tables
type Service ¶
type Service interface { //Handle retrieves windowed table from meta file and merges it with table info details Handle(*Request) *Response }
Service represents time windowed table service
type TableInfo ¶
type TableInfo struct { DatasetID string ProjectID string TableID string Created time.Time LastModified time.Time }
TableInfo represents table info
func GetTablesInfo ¶
GetTablesInfo returns table info for supplied dataset
func GetTablesInfoFromSQL ¶
type TimeWindow ¶
TimeWindow represents a table time winfow
type WindowedTable ¶
type WindowedTable struct { ID string ProjectID string Name string Dataset string Window *TimeWindow LastChanged time.Time Changed bool Expression string `description:"represents table ranged decorator table windowed expression"` AbsoluteExpression string `description:"represents absolute table path ranged decorator table windowed expression"` }
WindowedTable represents a windowed tables
func NewWindowedTable ¶
func NewWindowedTable(info *TableInfo, now time.Time) *WindowedTable
NewWindowedTable creates a new windowed table for supplied table info
func (*WindowedTable) FormatAbsoluteExpr ¶
func (t *WindowedTable) FormatAbsoluteExpr() string
FormatExpr formats form SQL range decorator expression
func (*WindowedTable) FormatExpr ¶
func (t *WindowedTable) FormatExpr() string
FormatExpr formats form SQL range decorator expression
func (*WindowedTable) FormatUnchangedAbsoluteExpr ¶
func (t *WindowedTable) FormatUnchangedAbsoluteExpr() string
FormatExpr formats form SQL range decorator expression
func (*WindowedTable) FormatUnchangedExpr ¶
func (t *WindowedTable) FormatUnchangedExpr() string