model

Documentation

Index

• Variables
• func AutoMigrate(db *gorm.DB) error
• func DecryptFromBase64String(payload string) ([]byte, error)
• func DecryptFromBytes(payload []byte) ([]byte, error)
• func DropAll(db *gorm.DB) error
• func EncryptToBase64String(payload []byte) (string, error)
• func EncryptToBytes(payload []byte) ([]byte, error)
• func EpochToTime(epoch int32) time.Time
• func EpochToUnix(epoch int32) int32
• func InitializeEncryption(keyStr string, db *gorm.DB) error
• func StoragePricePerEpochToPricePerDeal(price string, dealSize int64, durationEpoch int32) float64
• func UnixToEpoch(unix int64) int32
• type CID
  • func (c CID) MarshalJSON() ([]byte, error)
  • func (c *CID) Scan(src interface{}) error
  • func (c CID) String() string
  • func (c *CID) UnmarshalJSON(b []byte) error
  • func (c CID) Value() (driver.Value, error)
• type Car
• type CarBlock
  • func (c CarBlock) BlockLength() int32
• type Chunk
• type Dataset
• type Deal
  • func (d Deal) Key() string
• type DealState
• type Directory
• type Global
• type Item
• type Metadata
  • func (m *Metadata) Scan(src interface{}) error
  • func (m Metadata) Value() (driver.Value, error)
• type Schedule
  • func (s Schedule) Equal(other Schedule) bool
• type ScheduleState
• type Source
• type SourceType
• type StringSlice
  • func (ss *StringSlice) Scan(src interface{}) error
  • func (ss StringSlice) Value() (driver.Value, error)
• type Wallet
  • func (w Wallet) GetExportedKey() (string, error)
• type WalletAssignment
• type WorkState
• type WorkType
• type Worker

Variables

var DisableEncryption = false

var Tables = []interface{}{
	&Global{},
	&Worker{},
	&Dataset{},
	&Source{},
	&Chunk{},
	&Item{},
	&Directory{},
	&Car{},
	&CarBlock{},
	&Deal{},
	&Schedule{},
	&Wallet{},
	&WalletAssignment{},
}

Functions

func AutoMigrate

func AutoMigrate(db *gorm.DB) error

func DecryptFromBase64String

func DecryptFromBase64String(payload string) ([]byte, error)

func DecryptFromBytes

func DecryptFromBytes(payload []byte) ([]byte, error)

func DropAll

func DropAll(db *gorm.DB) error

func EncryptToBase64String

func EncryptToBase64String(payload []byte) (string, error)

func EncryptToBytes

func EncryptToBytes(payload []byte) ([]byte, error)

func EpochToTime

func EpochToTime(epoch int32) time.Time

func EpochToUnix

func EpochToUnix(epoch int32) int32

func InitializeEncryption

func InitializeEncryption(keyStr string, db *gorm.DB) error

func StoragePricePerEpochToPricePerDeal

func StoragePricePerEpochToPricePerDeal(price string, dealSize int64, durationEpoch int32) float64

func UnixToEpoch

func UnixToEpoch(unix int64) int32

Types

type CID

type CID cid.Cid

func (CID) MarshalJSON

func (c CID) MarshalJSON() ([]byte, error)

func (*CID) Scan

func (c *CID) Scan(src interface{}) error

func (CID) String

func (c CID) String() string

func (*CID) UnmarshalJSON

func (c *CID) UnmarshalJSON(b []byte) error

func (CID) Value

func (c CID) Value() (driver.Value, error)

type Car

type Car struct {
	ID        uint32    `gorm:"primaryKey" json:"id"`
	CreatedAt time.Time `json:"createdAt"`
	PieceCID  CID       `gorm:"column:piece_cid;index;type:bytes" json:"pieceCid"`
	PieceSize int64     `json:"pieceSize"`
	RootCID   CID       `gorm:"column:root_cid;type:bytes" json:"rootCid"`
	FileSize  int64     `json:"fileSize"`
	FilePath  string    `json:"filePath"`
	DatasetID uint32    `gorm:"index" json:"datasetId"`
	Dataset   *Dataset  `gorm:"foreignKey:DatasetID;constraint:OnDelete:CASCADE" json:"dataset,omitempty" swaggerignore:"true"`
	SourceID  *uint32   `gorm:"index" json:"sourceId"`
	Source    *Source   `gorm:"foreignKey:SourceID;constraint:OnDelete:CASCADE" json:"source,omitempty" swaggerignore:"true"`
	ChunkID   *uint32   `json:"chunkId"`
	Chunk     *Chunk    `gorm:"foreignKey:ChunkID;constraint:OnDelete:CASCADE" json:"chunk,omitempty" swaggerignore:"true"`
	Header    []byte    `json:"header"`
}

Car makes a reference to a CAR file that has been potentially exported to the disk. In the case of inline preparation, the path may be empty so the Car should be constructed on the fly using CarBlock, ItemBlock and RawBlock tables.

type CarBlock

type CarBlock struct {
	ID    uint64 `gorm:"primaryKey" json:"id"`
	CarID uint32 `json:"carId"`
	Car   *Car   `gorm:"foreignKey:CarID;constraint:OnDelete:CASCADE" json:"car,omitempty" swaggerignore:"true"`
	CID   CID    `gorm:"index;column:cid;type:bytes" json:"cid"`
	// Offset of the varint inside the CAR
	CarOffset      int64 `json:"carOffset"`
	CarBlockLength int32 `json:"carBlockLength"`
	// Value of the varint
	Varint []byte `json:"varint"`
	// Raw block
	RawBlock []byte `json:"rawBlock"`
	// If block is null, this block is a part of an item
	ItemID     *uint64 `json:"itemId"`
	Item       *Item   `gorm:"foreignKey:ItemID;constraint:OnDelete:CASCADE" json:"item,omitempty" swaggerignore:"true"`
	ItemOffset int64   `json:"itemOffset"`
}

CarBlock tells us the CIDs of all blocks inside a CAR file and the offset of the block inside the CAR file. From this table we can determine how to get the block by CID from a CAR file. or we can determine how to assemble a CAR file from blocks from original file.

func (CarBlock) BlockLength

func (c CarBlock) BlockLength() int32

type Chunk

type Chunk struct {
	ID              uint32    `gorm:"primaryKey" json:"id"`
	CreatedAt       time.Time `json:"createdAt"`
	SourceID        uint32    `json:"sourceId"`
	Source          *Source   `gorm:"foreignKey:SourceID;constraint:OnDelete:CASCADE" json:"source,omitempty" swaggerignore:"true"`
	PackingState    WorkState `json:"packingState"`
	PackingWorkerID *string   `json:"packingWorkerId,omitempty"`
	PackingWorker   *Worker   `` /* 128-byte string literal not displayed */
	ErrorMessage    string    `json:"errorMessage"`
	Items           []Item    `json:"items,omitempty" swaggerignore:"true"`
	Car             *Car      `json:"car,omitempty"`
}

Chunk is a grouping of items that are packed into a single CAR.

type Dataset

type Dataset struct {
	ID                   uint32      `gorm:"primaryKey" json:"id"`
	Name                 string      `gorm:"unique" json:"name"`
	CreatedAt            time.Time   `json:"createdAt"`
	UpdatedAt            time.Time   `json:"updatedAt"`
	MinSize              int64       `json:"minSize"`
	MaxSize              int64       `json:"maxSize"`
	PieceSize            int64       `json:"pieceSize"`
	OutputDirs           StringSlice `gorm:"type:JSON" json:"outputDirs"`
	EncryptionRecipients StringSlice `gorm:"type:JSON" json:"encryptionRecipients"`
	EncryptionScript     string      `json:"encryptionScript"`
	Wallets              []Wallet    `gorm:"many2many:wallet_assignments" json:"wallets,omitempty" swaggerignore:"true"`
}

Dataset is the top level object that represents a set of data to be onboarded.

type Deal

type Deal struct {
	ID           uint64 `gorm:"primaryKey"`
	CreatedAt    time.Time
	UpdatedAt    time.Time
	DealID       *uint64 `gorm:"unique"`
	DatasetID    *uint32
	Dataset      *Dataset  `gorm:"foreignKey:DatasetID;constraint:OnDelete:SET NULL"`
	State        DealState `gorm:"index:idx_stat"`
	ClientID     string
	Wallet       *Wallet `gorm:"foreignKey:ClientID;constraint:OnDelete:NO ACTION"`
	Provider     string  `gorm:"index:idx_stat"`
	ProposalID   string
	Label        string
	PieceCID     string `gorm:"column:piece_cid;index"`
	PieceSize    int64
	Start        time.Time
	Duration     time.Duration
	End          time.Time
	SectorStart  time.Time `gorm:"index:idx_stat"`
	Price        float64
	Verified     bool
	ErrorMessage string
	ScheduleID   *uint32
	Schedule     *Schedule
}

func (Deal) Key

func (d Deal) Key() string

type DealState

type DealState string

const (
	DealProposed        DealState = "proposed"
	DealPublished       DealState = "published"
	DealActive          DealState = "active"
	DealExpired         DealState = "expired"
	DealProposalExpired DealState = "proposal_expired"
	DealRejected        DealState = "rejected"
	DealSlashed         DealState = "slashed"
	DealErrored         DealState = "error"
)

type Directory

type Directory struct {
	ID       uint64     `gorm:"primaryKey" json:"id"`
	CID      CID        `gorm:"column:cid;type:bytes" json:"cid"`
	Data     []byte     `gorm:"column:data" json:"-" swaggerignore:"true"`
	Name     string     `json:"name"`
	ParentID *uint64    `gorm:"index" json:"parentId"`
	Parent   *Directory `gorm:"foreignKey:ParentID;constraint:OnDelete:CASCADE" json:"parent,omitempty" swaggerignore:"true"`
}

Directory is a link between parent and child directories.

type Global

type Global struct {
	Key   string `gorm:"primaryKey"`
	Value string
}

type Item

type Item struct {
	ID           uint64     `gorm:"primaryKey" json:"id"`
	ScannedAt    time.Time  `json:"scannedAt"`
	ChunkID      *uint32    `gorm:"index" json:"chunkId"`
	Chunk        *Chunk     `gorm:"foreignKey:ChunkID;constraint:OnDelete:CASCADE" json:"chunk,omitempty" swaggerignore:"true"`
	SourceID     uint32     `gorm:"index" json:"sourceId"`
	Source       *Source    `gorm:"foreignKey:SourceID;constraint:OnDelete:CASCADE" json:"source,omitempty" swaggerignore:"true"`
	Path         string     `json:"path"`
	Size         int64      `json:"size"`
	Offset       int64      `json:"offset"`
	Length       int64      `json:"length"`
	LastModified time.Time  `json:"lastModified"`
	CID          CID        `gorm:"column:cid;type:bytes" json:"cid"`
	DirectoryID  *uint64    `gorm:"index" json:"directoryId"`
	Directory    *Directory `gorm:"foreignKey:DirectoryID;constraint:OnDelete:CASCADE" json:"directory,omitempty" swaggerignore:"true"`
}

Item makes a reference to the data source item, i.e. a local file.

type Metadata

type Metadata map[string]string

func (*Metadata) Scan

func (m *Metadata) Scan(src interface{}) error

func (Metadata) Value

func (m Metadata) Value() (driver.Value, error)

type Schedule

type Schedule struct {
	ID                      uint32        `gorm:"primaryKey" json:"id"`
	CreatedAt               time.Time     `json:"createdAt"`
	UpdatedAt               time.Time     `json:"updatedAt"`
	DatasetID               uint32        `json:"datasetId"`
	Dataset                 *Dataset      `gorm:"foreignKey:DatasetID;constraint:OnDelete:CASCADE" json:"dataset,omitempty" swaggerignore:"true"`
	URLTemplate             string        `json:"urlTemplate"`
	HTTPHeaders             []string      `gorm:"type:JSON" json:"httpHeaders"`
	Provider                string        `json:"provider"`
	Price                   float64       `json:"price"`
	TotalDealNumber         int           `json:"totalDealNumber"`
	TotalDealSize           int64         `json:"totalDealSize"`
	Verified                bool          `json:"verified"`
	KeepUnsealed            bool          `json:"keepUnsealed"`
	AnnounceToIPNI          bool          `json:"announceToIPNI"`
	StartDelay              time.Duration `json:"startDelay"`
	Duration                time.Duration `json:"duration"`
	State                   ScheduleState `json:"state"`
	LastProcessedTimestamp  uint64        `json:"lastProcessedTimestamp"`
	ScheduleWorkerID        *string       `json:"scheduleWorkerId"`
	ScheduleWorker          *Worker       `gorm:"foreignKey:ScheduleWorkerID;constraint:OnDelete:NO ACTION" json:"scheduleWorker,omitempty" swaggerignore:"true"`
	ScheduleIntervalSeconds uint64        `json:"scheduleIntervalSeconds"`
	ScheduleDealNumber      int           `json:"scheduleDealNumber"`
	ScheduleDealSize        int64         `json:"scheduleDealSize"`
	MaxPendingDealNumber    int           `json:"maxPendingDealNumber"`
	MaxPendingDealSize      int64         `json:"maxPendingDealSize"`
	Notes                   string        `json:"notes"`
	ErrorMessage            string        `json:"errorMessage"`
	AllowedPieceCIDs        StringSlice   `json:"AllowedPieceCIDs"`
}

func (Schedule) Equal

func (s Schedule) Equal(other Schedule) bool

type ScheduleState

type ScheduleState string

const (
	ScheduleActive    ScheduleState = "active"
	SchedulePaused    ScheduleState = "paused"
	ScheduleCompleted ScheduleState = "completed"
)

type Source

type Source struct {
	ID                   uint32     `gorm:"primaryKey" json:"id"`
	CreatedAt            time.Time  `json:"createdAt"`
	UpdatedAt            time.Time  `json:"updatedAt"`
	DatasetID            uint32     `gorm:"uniqueIndex:dataset_type_path" json:"datasetId"`
	Dataset              *Dataset   `gorm:"foreignKey:DatasetID;constraint:OnDelete:CASCADE" json:"dataset,omitempty" swaggerignore:"true"`
	Type                 SourceType `gorm:"uniqueIndex:dataset_type_path" json:"type"`
	Path                 string     `gorm:"uniqueIndex:dataset_type_path" json:"path"`
	Metadata             Metadata   `gorm:"type:JSON" json:"metadata"`
	PushOnly             bool       `json:"pushOnly"`
	ScanIntervalSeconds  uint64     `json:"scanIntervalSeconds"`
	ScanningState        WorkState  `json:"scanningState"`
	ScanningWorkerID     *string    `json:"scanningWorkerId,omitempty"`
	ScanningWorker       *Worker    `` /* 130-byte string literal not displayed */
	LastScannedTimestamp int64      `json:"lastScannedTimestamp"`
	ErrorMessage         string     `json:"errorMessage"`
	RootDirectoryID      uint64     `json:"rootDirectoryId"`
	RootDirectory        *Directory `gorm:"foreignKey:RootDirectoryID;constraint:OnDelete:CASCADE" json:"rootDirectory,omitempty" swaggerignore:"true"`
	DeleteAfterExport    bool       `json:"deleteAfterExport"`
}

Source represents a source of data, i.e. a local file system directory.

type SourceType

type SourceType = string

const (
	Local  SourceType = "local"
	Upload SourceType = "upload"
)

type StringSlice

type StringSlice []string

func (*StringSlice) Scan

func (ss *StringSlice) Scan(src interface{}) error

func (StringSlice) Value

func (ss StringSlice) Value() (driver.Value, error)

type Wallet

type Wallet struct {
	ID         string `gorm:"primaryKey" json:"id"`  // ID is the short ID of the wallet
	Address    string `gorm:"unique" json:"address"` // Address is the Filecoin full address of the wallet
	PrivateKey string `json:"privateKey,omitempty"`  // PrivateKey is the private key of the wallet
	RemotePeer string `json:"remotePeer,omitempty"`  // RemotePeer is the remote peer ID of the wallet, for remote signing purpose
}

func (Wallet) GetExportedKey

func (w Wallet) GetExportedKey() (string, error)

type WalletAssignment

type WalletAssignment struct {
	ID        uint32 `gorm:"primaryKey"`
	WalletID  string
	Wallet    Wallet `gorm:"foreignKey:WalletID;constraint:OnDelete:CASCADE"`
	DatasetID uint32
	Dataset   Dataset `gorm:"foreignKey:DatasetID;constraint:OnDelete:CASCADE"`
}

type WorkState

type WorkState string

const (
	// Created means the item has been created is not ready for processing.
	Created WorkState = "created"
	// Ready means the item is ready to be processed.
	Ready WorkState = "ready"
	// Processing means the work is currently being processed.
	Processing WorkState = "processing"
	// Complete means the work is complete.
	Complete WorkState = "complete"
	// Error means the work has some error.
	Error WorkState = "error"
)

type WorkType

type WorkType string

const (
	Scan       WorkType = "scan"
	DealMaking WorkType = "deal_making"
	Packing    WorkType = "packing"
)

type Worker

type Worker struct {
	ID            string `gorm:"primaryKey"`
	WorkType      WorkType
	WorkingOn     string
	LastHeartbeat time.Time
	Hostname      string
}