storage

package
v0.0.0-...-d04bc5a Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Dec 24, 2024 License: Apache-2.0 Imports: 12 Imported by: 0

Documentation

Index

Constants

View Source 
const DistributedLockID = 100
View Source 
const (
	InsertBatchSize = 1000
)

Variables

This section is empty.

Functions

func CheckStorage 

func CheckStorage(store *gorm.DB) error

CheckStorage verifies storage is valid

func ColumnPrivs2ColumnControls 

func ColumnPrivs2ColumnControls(cps []ColumnPriv) []*pb.SecurityConfig_ColumnControl

Types

type Column 

type Column struct {
	ColumnIdentifier
	DType     string `gorm:"column:data_type;type:varchar(64);comment:'data type like float'"`
	CreatedAt time.Time
	UpdatedAt time.Time
}

type ColumnIdentifier 

type ColumnIdentifier struct {
	ProjectID  string `gorm:"column:project_id;type:varchar(64);primaryKey;not null"`
	TableName  string `gorm:"column:table_name;type:varchar(64);primaryKey;not null"`
	ColumnName string `gorm:"column:column_name;type:varchar(64);primaryKey;not null;"`
}

type ColumnMeta 

type ColumnMeta struct {
	ColumnName string
	DType      string
}

type ColumnPriv 

type ColumnPriv struct {
	ColumnPrivIdentifier
	Priv      string `gorm:"column:priv;type:varchar(256);comment:'priv of column'"`
	CreatedAt time.Time
	UpdatedAt time.Time
}

type ColumnPrivIdentifier 

type ColumnPrivIdentifier struct {
	ProjectID  string `gorm:"column:project_id;type:varchar(64);primaryKey;not null"`
	TableName  string `gorm:"column:table_name;type:varchar(64);primaryKey;not null"`
	ColumnName string `gorm:"column:column_name;type:varchar(64);primaryKey;not null;"`
	DestParty  string `gorm:"column:dest_party;type:varchar(64);primaryKey;not null;"`
}

type DistributeLockGuard 

type DistributeLockGuard struct {
	// contains filtered or unexported fields
}

func NewDistributeLockGuard 

func NewDistributeLockGuard(m *MetaManager) *DistributeLockGuard

Every time a lock operation is performed, a new guard must be used.

func (*DistributeLockGuard) DistributedLockExpired 

func (lm *DistributeLockGuard) DistributedLockExpired(id int8, owner string) (expired bool, err error)

NOTE: 1. InitDistributedLockIfNecessary must be called once before call DistributedLockExpired

  1. lock will expired after expired_at

func (*DistributeLockGuard) GetDistributedLockOwner 

func (lm *DistributeLockGuard) GetDistributedLockOwner(id int8) (owner string, err error)

func (*DistributeLockGuard) InitDistributedLockIfNecessary 

func (lm *DistributeLockGuard) InitDistributedLockIfNecessary(id int8) (err error)

func (*DistributeLockGuard) PreemptDistributedLock 

func (lm *DistributeLockGuard) PreemptDistributedLock(id int8, owner string, ttl time.Duration) (ok bool, err error)

func (*DistributeLockGuard) RenewDistributedLock 

func (lm *DistributeLockGuard) RenewDistributedLock(id int8, owner string, ttl time.Duration) (ok bool, err error)

type Invitation 

type Invitation struct {
	ID               uint64    `gorm:"column:id;primaryKey;comment:'auto generated increment id'"`
	ProjectID        string    `gorm:"column:project_id;type:varchar(64);not null;index:,composite:identifier;comment:'project id'"`
	Name             string    `gorm:"column:name;type:varchar(64);comment:'name'"`
	Description      string    `gorm:"column:description;type:varchar(64);comment:'description'"`
	Creator          string    `gorm:"column:creator;type:varchar(64);comment:'creator of the project'"`
	ProjectCreatedAt time.Time `gorm:"column:proj_created_at;comment:'the create time of the project'"`
	Member           string    `gorm:"column:member;type:string;not null;comment:'members, flattened string, like: alice;bob'"`
	ProjectConf      string    `gorm:"column:project_conf;type:text;comment:'project config in json format'"`
	Inviter          string    `gorm:"column:inviter;type:varchar(256);index:,composite:identifier;comment:'inviter'"`
	Invitee          string    `gorm:"column:invitee;type:varchar(256);index:,composite:identifier;comment:'invitee'"`
	// 0: default, not decided to accept invitation or not; 1: accepted; 2: rejected; 3: invalid
	Status     int8 `gorm:"column:status;default:0;comment:'accepted'"`
	InviteTime time.Time
	CreatedAt  time.Time
	UpdatedAt  time.Time
}

type Lock 

type Lock struct {
	ID        int8   `gorm:"column:id;primaryKey;uniqueIndex;comment:'lock id';->;<-:create"`
	Owner     string `gorm:"column:owner;type:varchar(64);comment:'lock owner'"`
	UpdatedAt time.Time
	ExpiredAt time.Time
}

type Member 

type Member struct {
	ProjectID string `gorm:"column:project_id;type:varchar(64);primaryKey;not null"`
	Member    string `gorm:"column:member;type:varchar(64);primaryKey;not null;comment:'member in the project'"`
}

type MetaManager 

type MetaManager struct {
	// contains filtered or unexported fields
}

func NewMetaManager 

func NewMetaManager(db *gorm.DB) *MetaManager

func (*MetaManager) Bootstrap 

func (manager *MetaManager) Bootstrap() error

Bootstrap init db

func (*MetaManager) CreateMetaTransaction 

func (manager *MetaManager) CreateMetaTransaction() *MetaTransaction

create a new MetaTransaction for every request and call Finish when you finish all your actions

func (*MetaManager) DropTables 

func (manager *MetaManager) DropTables() error

drop db for tests

func (*MetaManager) ExecInMetaTransaction 

func (manager *MetaManager) ExecInMetaTransaction(fn func(*MetaTransaction) error) error

func (*MetaManager) GetProject 

func (manager *MetaManager) GetProject(projectId string) (*Project, error)

func (*MetaManager) NeedBootstrap 

func (manager *MetaManager) NeedBootstrap() bool

NeedBootstrap checks if the store is empty

type MetaTransaction 

type MetaTransaction struct {
	// contains filtered or unexported fields
}

func AddExclusiveLock 

func AddExclusiveLock(txn *MetaTransaction) *MetaTransaction

func AddShareLock 

func AddShareLock(txn *MetaTransaction) *MetaTransaction

func (*MetaTransaction) AddInvitations 

func (t *MetaTransaction) AddInvitations(invitations []Invitation) error

duplicated invitations are permitted

func (*MetaTransaction) AddProjectMembers 

func (t *MetaTransaction) AddProjectMembers(members []Member) error

func (*MetaTransaction) AddTable 

func (t *MetaTransaction) AddTable(table TableMeta) error

func (*MetaTransaction) ArchiveProject 

func (t *MetaTransaction) ArchiveProject(projectID string) error

archive project fail if project not exists or other reasons

func (*MetaTransaction) CheckIdCanceled 

func (txn *MetaTransaction) CheckIdCanceled(ids []string) ([]string, error)

func (*MetaTransaction) CheckIdExpired 

func (txn *MetaTransaction) CheckIdExpired(ids []string) ([]string, error)

func (*MetaTransaction) ClearExpiredSessions 

func (txn *MetaTransaction) ClearExpiredSessions() (err error)

func (*MetaTransaction) ClearSessionInfo 

func (txn *MetaTransaction) ClearSessionInfo(id string) (err error)

func (*MetaTransaction) ClearSessionResult 

func (txn *MetaTransaction) ClearSessionResult(id string) (err error)

func (*MetaTransaction) CreateProject 

func (t *MetaTransaction) CreateProject(project Project) error

return err, if project exists

func (*MetaTransaction) DropTable 

func (t *MetaTransaction) DropTable(table TableIdentifier) error

DropTable will drop columns and table

func (*MetaTransaction) DropTableColumns 

func (t *MetaTransaction) DropTableColumns(table TableIdentifier) error

func (*MetaTransaction) Finish 

func (t *MetaTransaction) Finish(err error) error

Automatic rollback will occur if a deadlock or timeout occurs.

func (*MetaTransaction) GetInvitationsBy 

func (t *MetaTransaction) GetInvitationsBy(invitation Invitation, selectUnDecidedStatus bool) (invitations []Invitation, err error)

Note: because undecided status is zero, gorm will ignore it, so if select undecided invitations, set selectUnDecidedStatus true

func (*MetaTransaction) GetProject 

func (t *MetaTransaction) GetProject(projectID string) (Project, error)

func (*MetaTransaction) GetProjectAndMembers 

func (t *MetaTransaction) GetProjectAndMembers(projectID string) (projectAndMembers ProjectWithMember, err error)

func (*MetaTransaction) GetProjectMembers 

func (t *MetaTransaction) GetProjectMembers(projectID string) ([]string, error)

func (*MetaTransaction) GetProjectMeta 

func (t *MetaTransaction) GetProjectMeta(projectID string, tableNames []string, cclDestParties []string, owner string) (*ProjectMeta, error)

return all tables and ccls for the given project if tableNames and cclDestParties are nil

func (*MetaTransaction) GetSessionInfo 

func (txn *MetaTransaction) GetSessionInfo(id string) (*SessionInfo, error)

func (*MetaTransaction) GetSessionResult 

func (txn *MetaTransaction) GetSessionResult(id string) (SessionResult, error)

func (*MetaTransaction) GetSessionStatus 

func (txn *MetaTransaction) GetSessionStatus(id string) (SessionStatus, error)

func (*MetaTransaction) GetTableMetasByTableNames 

func (t *MetaTransaction) GetTableMetasByTableNames(projectID string, tableNames []string) (tableMetas []TableMeta, notFoundTables []string, err error)

if len(tableNames) == 0 return all tables return err if ANY table DOESN'T exist

func (*MetaTransaction) GetTables 

func (t *MetaTransaction) GetTables(projectID string, tableNames []string) (tables []Table, allTableExist bool, err error)

func (*MetaTransaction) GetUnhandledInvitation 

func (t *MetaTransaction) GetUnhandledInvitation(projectID, inviter, invitee string) (Invitation, error)

func (*MetaTransaction) GetUnhandledInvitationWithID 

func (t *MetaTransaction) GetUnhandledInvitationWithID(invitationID uint64) (Invitation, error)

func (*MetaTransaction) GetWatchedJobs 

func (txn *MetaTransaction) GetWatchedJobs() ([]*SessionInfo, error)

func (*MetaTransaction) GrantColumnConstraints 

func (t *MetaTransaction) GrantColumnConstraints(privs []ColumnPriv) error

func (*MetaTransaction) ListColumnConstraints 

func (t *MetaTransaction) ListColumnConstraints(projectID string, tableNames []string, destParties []string) ([]ColumnPriv, error)

func (*MetaTransaction) ListDedupTableOwners 

func (t *MetaTransaction) ListDedupTableOwners(projectID string, tableNames []string) ([]string, error)

func (*MetaTransaction) ListInvitations 

func (t *MetaTransaction) ListInvitations() ([]Invitation, error)

func (*MetaTransaction) ListProjects 

func (t *MetaTransaction) ListProjects(projectIDs []string) ([]ProjectWithMember, error)

func (*MetaTransaction) ModifyInvitationStatus 

func (t *MetaTransaction) ModifyInvitationStatus(invitationID uint64, status pb.InvitationStatus) error

NOTE: invitation id may be zero don't use Where(&Invitation{ID: id})

func (*MetaTransaction) RevokeColumnConstraints 

func (t *MetaTransaction) RevokeColumnConstraints(privIdentifiers []ColumnPrivIdentifier) error

use when ccl changed

func (*MetaTransaction) SetInvitationInvalidByID 

func (t *MetaTransaction) SetInvitationInvalidByID(invitationID uint64) error

func (*MetaTransaction) SetMultipleSessionStatus 

func (txn *MetaTransaction) SetMultipleSessionStatus(ids []string, status SessionStatus) error

func (*MetaTransaction) SetSessionInfo 

func (txn *MetaTransaction) SetSessionInfo(info *SessionInfo) error

func (*MetaTransaction) SetSessionResult 

func (txn *MetaTransaction) SetSessionResult(sr SessionResult) (err error)

NOTE: check SessionInfo exist before calling

func (*MetaTransaction) SetSessionStatus 

func (txn *MetaTransaction) SetSessionStatus(id string, status SessionStatus) error

func (*MetaTransaction) SetUnhandledInvitationsInvalid 

func (t *MetaTransaction) SetUnhandledInvitationsInvalid(projectID, inviter, invitee string) error

func (*MetaTransaction) UpdateProject 

func (t *MetaTransaction) UpdateProject(proj Project) error

update project fail if project not exists or other reasons

func (*MetaTransaction) UpdateSessionInfo 

func (txn *MetaTransaction) UpdateSessionInfo(info *SessionInfo) error

func (*MetaTransaction) UpdateSessionInfoStatusWithCondition 

func (txn *MetaTransaction) UpdateSessionInfoStatusWithCondition(sessionID string, conditionStatus, targetStatus SessionStatus) error

func (*MetaTransaction) UpdateSessionUpdatedAt 

func (txn *MetaTransaction) UpdateSessionUpdatedAt(ids []string) error

type Project 

type Project struct {
	// ->;<-:create means read and create
	// id can't be modified
	ID          string `gorm:"column:id;type:varchar(64);primaryKey;uniqueIndex:;comment:'unique id';->;<-:create"`
	Name        string `gorm:"column:name;type:varchar(64);not null;comment:'project name'"`
	Description string `gorm:"column:description;type:varchar(64);comment:'description'"`
	Creator     string `gorm:"column:creator;type:varchar(64);comment:'creator of the project'"`
	Archived    bool   `gorm:"column:archived;comment:'if archived is true, whole project can't be modified'"`
	ProjectConf string `gorm:"column:project_conf;type:text;comment:'project config in json format'"`
	CreatedAt   time.Time
	UpdatedAt   time.Time
}

func (*Project) Equals 

func (p *Project) Equals(other *Project) (bool, error)

type ProjectMeta 

type ProjectMeta struct {
	Proj   ProjectWithMember
	Tables []TableMeta
	CCLs   []ColumnPriv
}

type ProjectWithMember 

type ProjectWithMember struct {
	Proj    Project
	Members []string
}

type SessionInfo 

type SessionInfo struct {
	SessionID string `gorm:"column:session_id;type:varchar(64);primaryKey;uniqueIndex:;comment:'unique session id';->;<-:create"`
	// 0: default, running; 1: finished; 2: canceled
	Status           int8   `gorm:"column:status;default:0;comment:'session status'"`
	TableChecksum    []byte `gorm:"column:table_checksum;type:varbinary(256);comment:'table checksum for self party'"`
	CCLChecksum      []byte `gorm:"column:ccl_checksum;type:varbinary(256);comment:'ccl checksum for self party'"`
	EngineUrl        string `gorm:"column:engine_url;type:varchar(256);comment:'url for engine to communicate with peer engine'"`
	EngineUrlForSelf string `gorm:"column:engine_url_for_self;type:varchar(256);comment:'engine url used for self broker'"`
	JobInfo          []byte `gorm:"column:job_info;type:bytes;comment:'serialized job info to specify task in engine'"`
	WorkParties      string `gorm:"column:work_parties;type:string;not null;comment:'parties involved, flattened string, like: alice;bob'"`
	OutputNames      string `gorm:"column:output_names;type:string;comment:'output column names, flattened string, like: col1,col2'"`
	Warning          []byte `gorm:"column:warning;type:bytes;comment:'warning infos, serialized from pb.Warning'"`
	CreatedAt        time.Time
	UpdatedAt        time.Time
	ExpiredAt        time.Time
}

SessionInfo and SessionResult ares used to support broker cluster mode

type SessionResult 

type SessionResult struct {
	SessionID string `gorm:"column:session_id;type:varchar(64);primaryKey;uniqueIndex:;comment:'unique session id';->;<-:create"`
	Result    []byte `gorm:"column:result;type:bytes;comment:'query result, serialized from protobuf message'"`
	CreatedAt time.Time
	UpdatedAt time.Time
	ExpiredAt time.Time
}

type SessionStatus 

type SessionStatus int8
const (
	SessionRunning SessionStatus = iota
	SessionFinished
	SessionCanceled
	SessionTimeout
	SessionFailed
	SessionSubmitted
)

This is by design

type Table 

type Table struct {
	TableIdentifier
	RefTable string `gorm:"column:ref_table;type:varchar(128);comment:'ref table'"`
	DBType   string `gorm:"column:db_type;type:varchar(64);comment:'database type like MYSQL'"`
	Owner    string `gorm:"column:owner;comment:'table owner'"`
	// view
	IsView       bool   `gorm:"column:is_view;comment:'this table is a view'"`
	SelectString string `gorm:"column:select_string;comment:'the internal select query in string format, the field is valid only when IsView is true'"`

	CreatedAt time.Time
	UpdatedAt time.Time
}

type TableIdentifier 

type TableIdentifier struct {
	ProjectID string `gorm:"column:project_id;type:varchar(64);primaryKey;not null"`
	TableName string `gorm:"column:table_name;type:varchar(64);primaryKey;not null"`
}

type TableMeta 

type TableMeta struct {
	Table   Table
	Columns []ColumnMeta
}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.