model

Documentation

Index

• type AuthPolicyInput
• type AuthPolicyOutput
• type BigQueryDest
• type CloudStorageEvent
  • func (x CloudStorageEvent) ToObject() Object
• type CloudStorageObject
• type Digest
• type EnqueueRequest
• type EnqueueResponse
• type EventPolicyOutput
• type EventarcDirectEvent
• type IngestLog
  • func (x *IngestLog) Raw() *IngestLogRaw
• type IngestLogRaw
• type LoadDataRequest
• type LoadLog
  • func (x *LoadLog) Raw() *LoadLogRaw
• type LoadLogRaw
• type LoadRequest
• type Log
  • func (x *Log) Validate() error
• type LogRecord
  • func (x LogRecord) Raw() *LogRecordRaw
• type LogRecordRaw
• type LogRecordSet
  • func (x LogRecordSet) Merge(src LogRecordSet)
• type MetadataConfig
  • func NewMetadataConfig(dataset types.BQDatasetID, table types.BQTableID) *MetadataConfig
  • func (x *MetadataConfig) Dataset() types.BQDatasetID
  • func (x *MetadataConfig) Table() types.BQTableID
• type Object
  • func NewObjectFromCloudStorageAttrs(attrs *storage.ObjectAttrs) Object
• type PubSubBody
• type PubSubMessage
• type SchemaPolicyOutput
• type Source
  • func (x Source) Validate() error
• type SourceLog
  • func (x *SourceLog) Raw() *SourceLogRaw
• type SourceLogRaw
• type State
  • func (x *State) Acquired(now time.Time) bool
• type SwarmMessage

Types

type AuthPolicyInput

type AuthPolicyInput struct {
	Method string              `json:"method"`
	Path   string              `json:"path"`
	Remote string              `json:"remote"`
	Query  map[string][]string `json:"query"`
	Header map[string][]string `json:"header"`
	Body   string              `json:"body"`
}

type AuthPolicyOutput

type AuthPolicyOutput struct {
	Deny bool `json:"deny"`
}

type BigQueryDest

type BigQueryDest struct {
	Dataset   types.BQDatasetID `json:"dataset"`
	Table     types.BQTableID   `json:"table"`
	Partition types.BQPartition `json:"partition"`
}

type CloudStorageEvent

type CloudStorageEvent struct {
	Bucket                  types.CSBucket   `json:"bucket"`
	ContentType             string           `json:"contentType"`
	Crc32c                  string           `json:"crc32c"`
	Etag                    string           `json:"etag"`
	Generation              string           `json:"generation"`
	ID                      string           `json:"id"`
	Kind                    string           `json:"kind"`
	Md5Hash                 string           `json:"md5Hash"`
	MediaLink               string           `json:"mediaLink"`
	Metageneration          string           `json:"metageneration"`
	Name                    types.CSObjectID `json:"name"`
	SelfLink                string           `json:"selfLink"`
	Size                    string           `json:"size"`
	StorageClass            string           `json:"storageClass"`
	TimeCreated             string           `json:"timeCreated"`
	TimeStorageClassUpdated string           `json:"timeStorageClassUpdated"`
	Updated                 string           `json:"updated"`
}

func (CloudStorageEvent) ToObject

func (x CloudStorageEvent) ToObject() Object

type CloudStorageObject

type CloudStorageObject struct {
	Bucket types.CSBucket   `json:"bucket" bigquery:"bucket"`
	Name   types.CSObjectID `json:"name" bigquery:"name"`
}

type Digest

type Digest struct {
	Alg   string `json:"alg" bigquery:"alg"`
	Value string `json:"value" bigquery:"value"`
}

type EnqueueRequest

type EnqueueRequest struct {
	URLs []types.ObjectURL
}

type EnqueueResponse

type EnqueueResponse struct {
	Elapsed time.Duration
	Count   int64
	Size    int64
}

type EventPolicyOutput

type EventPolicyOutput struct {
	Sources []*Source `json:"src"`
}

type EventarcDirectEvent

type EventarcDirectEvent struct {
	Bucket                  types.CSBucket   `json:"bucket"`
	ContentType             string           `json:"contentType"`
	Crc32c                  string           `json:"crc32c"`
	Etag                    string           `json:"etag"`
	Generation              string           `json:"generation"`
	ID                      string           `json:"id"`
	Kind                    string           `json:"kind"`
	Md5Hash                 string           `json:"md5Hash"`
	MediaLink               string           `json:"mediaLink"`
	Metageneration          string           `json:"metageneration"`
	Name                    types.CSObjectID `json:"name"`
	SelfLink                string           `json:"selfLink"`
	Size                    string           `json:"size"`
	StorageClass            string           `json:"storageClass"`
	TimeCreated             string           `json:"timeCreated"`
	TimeStorageClassUpdated string           `json:"timeStorageClassUpdated"`
	Updated                 string           `json:"updated"`
}

type IngestLog

type IngestLog struct {
	ID           types.IngestID     `json:"id" bigquery:"id"`
	StartedAt    time.Time          `json:"started_at" bigquery:"started_at"`
	FinishedAt   time.Time          `json:"finished_at" bigquery:"finished_at"`
	ObjectSchema types.ObjectSchema `json:"object_schema" bigquery:"object_schema"`
	DatasetID    types.BQDatasetID  `json:"dataset_id" bigquery:"dataset_id"`
	TableID      types.BQTableID    `json:"table_id" bigquery:"table_id"`
	TableSchema  string             `json:"table_schema" bigquery:"table_schema"`
	LogCount     int                `json:"log_count" bigquery:"log_count"`
	Success      bool               `json:"success" bigquery:"success"`
	Error        string             `json:"error" bigquery:"error"`
}

func (*IngestLog) Raw

func (x *IngestLog) Raw() *IngestLogRaw

type IngestLogRaw

type IngestLogRaw struct {
	IngestLog
	StartedAt  int64 `json:"started_at" bigquery:"started_at"`
	FinishedAt int64 `json:"finished_at" bigquery:"finished_at"`
}

type LoadDataRequest

type LoadDataRequest struct {
	CSEvent *CloudStorageEvent
}

type LoadLog

type LoadLog struct {
	ID         types.RequestID `json:"id" bigquery:"id"`
	StartedAt  time.Time       `json:"started_at" bigquery:"started_at"`
	FinishedAt time.Time       `json:"finished_at" bigquery:"finished_at"`
	Success    bool            `json:"success" bigquery:"success"`
	Sources    []*SourceLog    `json:"sources" bigquery:"sources"`
	Ingests    []*IngestLog    `json:"ingests" bigquery:"ingests"`
	Error      string          `json:"error" bigquery:"error"`
}

func (*LoadLog) Raw

func (x *LoadLog) Raw() *LoadLogRaw

type LoadLogRaw

type LoadLogRaw struct {
	LoadLog
	StartedAt  int64           `json:"started_at" bigquery:"started_at"`
	FinishedAt int64           `json:"finished_at" bigquery:"finished_at"`
	Ingests    []*IngestLogRaw `json:"ingests" bigquery:"ingests"`
	Sources    []*SourceLogRaw `json:"sources" bigquery:"sources"`
}

type LoadRequest

type LoadRequest struct {
	Source Source
	Object Object
}

type Log

type Log struct {
	// Destination BigQuery table information
	BigQueryDest

	ID        types.LogID    `json:"id"`
	Timestamp float64        `json:"timestamp"`
	Data      map[string]any `json:"data"`
}

func (*Log) Validate

func (x *Log) Validate() error

type LogRecord

type LogRecord struct {
	// NOTICE: Must update LogRecordRaw also when adding new fields to LogRecord
	ID         types.LogID    `json:"id" bigquery:"id"`
	IngestID   types.IngestID `json:"ingest_id" bigquery:"ingest_id"`
	Timestamp  time.Time      `json:"timestamp" bigquery:"timestamp"`
	IngestedAt time.Time      `json:"ingested_at" bigquery:"ingested_at"`
	Data       any            `json:"data" bigquery:"data"`
}

func (LogRecord) Raw

func (x LogRecord) Raw() *LogRecordRaw

type LogRecordRaw

type LogRecordRaw struct {
	LogRecord
	Timestamp  int64 `json:"timestamp" bigquery:"timestamp"`
	IngestedAt int64 `json:"ingested_at" bigquery:"ingested_at"`
}

LogRecordRaw is replaced LogRecord with Timestamp from time.Time to int64. BigQuery Storage Write API requires converting data to protocol buffer. But adapt.StorageSchemaToProto2Descriptor is not supported for time.Time. It uses int64 for timestamp instead of time.Time. So, LogRecordRaw is used for only insertion by BigQuery Storage Write API.

type LogRecordSet

type LogRecordSet map[BigQueryDest][]*LogRecord

func (LogRecordSet) Merge

func (x LogRecordSet) Merge(src LogRecordSet)

type MetadataConfig

type MetadataConfig struct {
	// contains filtered or unexported fields
}

func NewMetadataConfig

func NewMetadataConfig(dataset types.BQDatasetID, table types.BQTableID) *MetadataConfig

func (*MetadataConfig) Dataset

func (x *MetadataConfig) Dataset() types.BQDatasetID

func (*MetadataConfig) Table

func (x *MetadataConfig) Table() types.BQTableID

type Object

type Object struct {
	CS        *CloudStorageObject `json:"cs,omitempty" bigquery:"cs"`
	Size      *int64              `json:"size,omitempty" bigquery:"size"`
	CreatedAt *int64              `json:"created_at" bigquery:"created_at"`
	Digests   []Digest            `json:"digests" bigquery:"digests"`

	// Data is original notification data, such as CloudStorageEvent
	Data any `json:"data" bigquery:"-"`
}

func NewObjectFromCloudStorageAttrs

func NewObjectFromCloudStorageAttrs(attrs *storage.ObjectAttrs) Object

type PubSubBody

type PubSubBody struct {
	Message      PubSubMessage `json:"message"`
	Subscription string        `json:"subscription"`
}

type PubSubMessage

type PubSubMessage struct {
	Attributes  map[string]string `json:"attributes"`
	Data        string            `json:"data"`
	MessageID   string            `json:"message_id"`
	PublishTime string            `json:"publish_time"`
}

type SchemaPolicyOutput

type SchemaPolicyOutput struct {
	Logs []*Log `json:"log"`
}

type Source

type Source struct {
	// Source object information
	Parser   types.ObjectParser   `json:"parser" bigquery:"parser"`
	Schema   types.ObjectSchema   `json:"schema" bigquery:"schema"`
	Compress types.ObjectCompress `json:"compress" bigquery:"compress"`
}

func (Source) Validate

func (x Source) Validate() error

type SourceLog

type SourceLog struct {
	CS         *CloudStorageObject `json:"cs" bigquery:"cs"`
	Source     Source              `json:"source" bigquery:"source"`
	RowCount   int                 `json:"row_count" bigquery:"row_count"`
	StartedAt  time.Time           `json:"started_at" bigquery:"started_at"`
	FinishedAt time.Time           `json:"finished_at" bigquery:"finished_at"`
	Success    bool                `json:"success" bigquery:"success"`
}

func (*SourceLog) Raw

func (x *SourceLog) Raw() *SourceLogRaw

type SourceLogRaw

type SourceLogRaw struct {
	SourceLog
	StartedAt  int64 `json:"started_at" bigquery:"started_at"`
	FinishedAt int64 `json:"finished_at" bigquery:"finished_at"`
}

type State

type State struct {
	ID        string          `firestore:"id"`
	RequestID types.RequestID `firestore:"request_id"`
	State     types.MsgState  `firestore:"state"`
	CreatedAt time.Time       `firestore:"created_at"`
	UpdatedAt time.Time       `firestore:"updated_at"`
	ExpiresAt time.Time       `firestore:"expires_at"`
	TTL       time.Time       `firestore:"ttl"`
}

func (*State) Acquired

func (x *State) Acquired(now time.Time) bool

type SwarmMessage

type SwarmMessage struct {
	Objects []*Object `json:"objects"`
}

SwarmMessage is a struct for the event from swarm. It's abstracted event structure for multiple event sources.