model

package
v0.2.47 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jul 26, 2023 License: Apache-2.0, MIT Imports: 13 Imported by: 1

Documentation

Index

Constants

This section is empty.

Variables

View Source 
var DealStates = []DealState{
	DealProposed,
	DealPublished,
	DealActive,
	DealExpired,
	DealProposalExpired,
	DealRejected,
	DealSlashed,
	DealErrored,
}
View Source 
var ScheduleStates = []ScheduleState{
	ScheduleActive,
	SchedulePaused,
	ScheduleError,
	ScheduleCompleted,
}
View Source 
var Tables = []any{
	&Global{},
	&Worker{},
	&Dataset{},
	&Source{},
	&Chunk{},
	&Item{},
	&Directory{},
	&Car{},
	&CarBlock{},
	&Deal{},
	&Schedule{},
	&Wallet{},
	&WalletAssignment{},
	&ItemPart{},
}
View Source 
var WorkStates = []WorkState{
	Created,
	Ready,
	Processing,
	Complete,
	Error,
}
View Source 
var WorkTypes = []WorkType{
	Scan,
	DealMaking,
	DealTracking,
	Packing,
}

Functions

func AutoMigrate 

func AutoMigrate(db *gorm.DB) error

func DropAll 

func DropAll(db *gorm.DB) error

func EpochToTime 

func EpochToTime(epoch int32) time.Time

func EpochToUnix 

func EpochToUnix(epoch int32) int32

func StoragePricePerEpochToPricePerDeal 

func StoragePricePerEpochToPricePerDeal(price string, dealSize int64, durationEpoch int32) float64

func UnixToEpoch 

func UnixToEpoch(unix int64) int32

Types

type CID 

type CID cid.Cid

func (CID) MarshalBinary added in v0.2.14 

func (c CID) MarshalBinary() ([]byte, error)

func (CID) MarshalJSON 

func (c CID) MarshalJSON() ([]byte, error)

func (*CID) Scan 

func (c *CID) Scan(src any) error

func (CID) String 

func (c CID) String() string

func (*CID) UnmarshalBinary added in v0.2.14 

func (c *CID) UnmarshalBinary(b []byte) error

func (*CID) UnmarshalJSON 

func (c *CID) UnmarshalJSON(b []byte) error

func (CID) Value 

func (c CID) Value() (driver.Value, error)

type Car 

type Car struct {
	ID        uint32    `gorm:"primaryKey"                                       json:"id"`
	CreatedAt time.Time `json:"createdAt"`
	PieceCID  CID       `gorm:"column:piece_cid;index;type:bytes;size:256"       json:"pieceCid"`
	PieceSize int64     `json:"pieceSize"`
	RootCID   CID       `gorm:"column:root_cid;type:bytes;size:256"              json:"rootCid"`
	FileSize  int64     `json:"fileSize"`
	FilePath  string    `gorm:"size:1024"                                        json:"filePath"`
	DatasetID uint32    `gorm:"index"                                            json:"datasetId"`
	Dataset   *Dataset  `gorm:"foreignKey:DatasetID;constraint:OnDelete:CASCADE" json:"dataset,omitempty" swaggerignore:"true"`
	SourceID  *uint32   `gorm:"index"                                            json:"sourceId"`
	Source    *Source   `gorm:"foreignKey:SourceID;constraint:OnDelete:CASCADE"  json:"source,omitempty"  swaggerignore:"true"`
	ChunkID   *uint32   `json:"chunkId"`
	Chunk     *Chunk    `gorm:"foreignKey:ChunkID;constraint:OnDelete:CASCADE"   json:"chunk,omitempty"   swaggerignore:"true"`
	Header    []byte    `gorm:"size:256"                                         json:"header"`
	// contains filtered or unexported fields
}

Car makes a reference to a CAR file that has been potentially exported to the disk. In the case of inline preparation, the path may be empty so the Car should be constructed on the fly using CarBlock, ItemBlock and RawBlock tables.

type CarBlock 

type CarBlock struct {
	ID    uint64 `gorm:"primaryKey"                                   json:"id"`
	CarID uint32 `json:"carId"`
	Car   *Car   `gorm:"foreignKey:CarID;constraint:OnDelete:CASCADE" json:"car,omitempty" swaggerignore:"true"`
	CID   CID    `gorm:"index;column:cid;type:bytes;size:256"         json:"cid"`
	// Offset of the varint inside the CAR
	CarOffset      int64 `json:"carOffset"`
	CarBlockLength int32 `json:"carBlockLength"`
	// Value of the varint
	Varint []byte `json:"varint"`
	// Raw block
	RawBlock []byte `json:"rawBlock"`
	// If block is null, this block is a part of an item
	ItemID *uint64 `json:"itemId"`
	Item   *Item   `gorm:"foreignKey:ItemID;constraint:OnDelete:CASCADE" json:"item,omitempty" swaggerignore:"true"`
	// A reference to the item with offset. Meaningless if item is encrypted since it's the offset of the encrypted object.
	ItemOffset    int64 `json:"itemOffset"`
	ItemEncrypted bool  `json:"itemEncrypted"`
	// contains filtered or unexported fields
}

CarBlock tells us the CIDs of all blocks inside a CAR file and the offset of the block inside the CAR file. From this table we can determine how to get the block by CID from a CAR file. or we can determine how to assemble a CAR file from blocks from original file.

func (CarBlock) BlockLength 

func (c CarBlock) BlockLength() int32

type Chunk 

type Chunk struct {
	ID              uint32     `gorm:"primaryKey"                                                            json:"id"`
	CreatedAt       time.Time  `json:"createdAt"`
	SourceID        uint32     `gorm:"index:source_summary_chunks"                                           json:"sourceId"`
	Source          *Source    `` /* 130-byte string literal not displayed */
	PackingState    WorkState  `gorm:"index:source_summary_chunks;index:chunk_cleanup;size:16"               json:"packingState"`
	PackingWorkerID *string    `gorm:"index:chunk_cleanup"                                                   json:"packingWorkerId,omitempty"`
	PackingWorker   *Worker    `` /* 130-byte string literal not displayed */
	ErrorMessage    string     `json:"errorMessage"`
	ItemParts       []ItemPart `gorm:"constraint:OnDelete:SET NULL"                                          json:"itemParts,omitempty"`
	Cars            []Car      `gorm:"constraint:OnDelete:CASCADE"                                           json:"cars,omitempty"`
}

Chunk is a grouping of items that are packed into a single CAR.

type Dataset 

type Dataset struct {
	ID                   uint32      `gorm:"primaryKey"                   json:"id"`
	Name                 string      `gorm:"unique"                       json:"name"`
	CreatedAt            time.Time   `json:"createdAt"`
	UpdatedAt            time.Time   `json:"updatedAt"`
	MaxSize              int64       `json:"maxSize"`
	PieceSize            int64       `json:"pieceSize"`
	OutputDirs           StringSlice `gorm:"type:JSON"                    json:"outputDirs"`
	EncryptionRecipients StringSlice `gorm:"type:JSON"                    json:"encryptionRecipients"`
	EncryptionScript     string      `json:"encryptionScript"`
	Wallets              []Wallet    `gorm:"many2many:wallet_assignments" json:"wallets,omitempty"    swaggerignore:"true"`
}

Dataset is the top level object that represents a set of data to be onboarded.

func (Dataset) UseEncryption added in v0.1.0 

func (d Dataset) UseEncryption() bool

type Deal 

type Deal struct {
	ID               uint64    `gorm:"primaryKey"                                         json:"id"`
	CreatedAt        time.Time `json:"createdAt"`
	UpdatedAt        time.Time `json:"updatedAt"`
	DealID           *uint64   `gorm:"unique"                                             json:"dealId"`
	DatasetID        *uint32   `json:"datasetId"`
	Dataset          *Dataset  `gorm:"foreignKey:DatasetID;constraint:OnDelete:SET NULL"  json:"dataset,omitempty"  swaggerignore:"true"`
	State            DealState `gorm:"index:idx_stat;index:idx_pending;size:16"           json:"state"`
	ClientID         string    `gorm:"index:idx_pending;size:16"                          json:"clientId"`
	Wallet           *Wallet   `gorm:"foreignKey:ClientID;constraint:OnDelete:SET NULL"   json:"wallet,omitempty"   swaggerignore:"true"`
	Provider         string    `gorm:"index:idx_stat;size:16"                             json:"provider"`
	ProposalID       string    `json:"proposalId"`
	Label            string    `json:"label"`
	PieceCID         CID       `gorm:"column:piece_cid;index"                             json:"pieceCid"`
	PieceSize        int64     `json:"pieceSize"`
	StartEpoch       int32     `json:"startEpoch"`
	EndEpoch         int32     `json:"endEpoch"`
	SectorStartEpoch int32     `json:"sectorStartEpoch"`
	Price            string    `json:"price"`
	Verified         bool      `gorm:"index:idx_pending"                                  json:"verified"`
	ErrorMessage     string    `json:"errorMessage"`
	ScheduleID       *uint32   `json:"scheduleId"`
	Schedule         *Schedule `gorm:"foreignKey:ScheduleID;constraint:OnDelete:SET NULL" json:"schedule,omitempty" swaggerignore:"true"`
}

func (Deal) Key 

func (d Deal) Key() string

type DealState 

type DealState string
const (
	DealProposed        DealState = "proposed"
	DealPublished       DealState = "published"
	DealActive          DealState = "active"
	DealExpired         DealState = "expired"
	DealProposalExpired DealState = "proposal_expired"
	DealRejected        DealState = "rejected"
	DealSlashed         DealState = "slashed"
	DealErrored         DealState = "error"
)

type Directory 

type Directory struct {
	ID        uint64     `gorm:"primaryKey"                                      json:"id"`
	UpdatedAt time.Time  `json:"updatedAt"`
	CID       CID        `gorm:"column:cid;type:bytes;size:256"                  json:"cid"`
	SourceID  uint32     `gorm:"index"                                           json:"sourceId"`
	Source    *Source    `gorm:"foreignKey:SourceID;constraint:OnDelete:CASCADE" json:"source,omitempty" swaggerignore:"true"`
	Data      []byte     `gorm:"column:data"                                     json:"-"                swaggerignore:"true"`
	Name      string     `gorm:"size:256"                                        json:"name"`
	ParentID  *uint64    `gorm:"index"                                           json:"parentId"`
	Parent    *Directory `gorm:"foreignKey:ParentID;constraint:OnDelete:CASCADE" json:"parent,omitempty" swaggerignore:"true"`
	Exported  bool       `json:"exported"`
}

Directory is a link between parent and child directories.

type Global 

type Global struct {
	Key   string `gorm:"primaryKey;size:256" json:"key"`
	Value string `json:"value"`
}

type Item 

type Item struct {
	ID                        uint64     `gorm:"primaryKey"                                                json:"id"`
	CreatedAt                 time.Time  `json:"createdAt"`
	SourceID                  uint32     `gorm:"index:check_existence;index:source_summary_items"          json:"sourceId"`
	Source                    *Source    `gorm:"foreignKey:SourceID;constraint:OnDelete:CASCADE"           json:"source,omitempty"    swaggerignore:"true"`
	Path                      string     `gorm:"index:check_existence;size:1024"                           json:"path"`
	Hash                      string     `gorm:"index:check_existence;size:256"                            json:"hash"`
	Size                      int64      `gorm:"index:check_existence"                                     json:"size"`
	LastModifiedTimestampNano int64      `gorm:"index:check_existence"                                     json:"lastModified"`
	CID                       CID        `gorm:"index:source_summary_items;column:cid;type:bytes;size:256" json:"cid"`
	DirectoryID               *uint64    `gorm:"index"                                                     json:"directoryId"`
	Directory                 *Directory `gorm:"foreignKey:DirectoryID;constraint:OnDelete:CASCADE"        json:"directory,omitempty" swaggerignore:"true"`
	ItemParts                 []ItemPart `gorm:"constraint:OnDelete:CASCADE"                               json:"itemParts,omitempty"`
	// contains filtered or unexported fields
}

Item makes a reference to the data source item, i.e. a local file.

type ItemPart added in v0.1.0 

type ItemPart struct {
	ID      uint64  `gorm:"primaryKey"                                      json:"id"`
	ItemID  uint64  `gorm:"index:find_remaining"                            json:"itemId"`
	Item    *Item   `gorm:"foreignKey:ItemID;constraint:OnDelete:CASCADE"   json:"item,omitempty"`
	Offset  int64   `json:"offset"`
	Length  int64   `json:"length"`
	CID     CID     `gorm:"column:cid;type:bytes;size:256"                  json:"cid"`
	ChunkID *uint32 `gorm:"index:find_remaining"                            json:"chunkId"`
	Chunk   *Chunk  `gorm:"foreignKey:ChunkID;constraint:OnDelete:SET NULL" json:"chunk,omitempty" swaggerignore:"true"`
}

type Metadata 

type Metadata map[string]string

func (*Metadata) Scan 

func (m *Metadata) Scan(src any) error

func (Metadata) Value 

func (m Metadata) Value() (driver.Value, error)

type Schedule 

type Schedule struct {
	ID                   uint32        `gorm:"primaryKey"                                       json:"id"`
	CreatedAt            time.Time     `json:"createdAt"`
	UpdatedAt            time.Time     `json:"updatedAt"`
	DatasetID            uint32        `json:"datasetId"`
	Dataset              *Dataset      `gorm:"foreignKey:DatasetID;constraint:OnDelete:CASCADE" json:"dataset,omitempty" swaggerignore:"true"`
	URLTemplate          string        `json:"urlTemplate"`
	HTTPHeaders          []string      `gorm:"type:JSON"                                        json:"httpHeaders"`
	Provider             string        `json:"provider"`
	PricePerGBEpoch      float64       `json:"pricePerGbEpoch"`
	PricePerGB           float64       `json:"pricePerGb"`
	PricePerDeal         float64       `json:"pricePerDeal"`
	TotalDealNumber      int           `json:"totalDealNumber"`
	TotalDealSize        int64         `json:"totalDealSize"`
	Verified             bool          `json:"verified"`
	KeepUnsealed         bool          `json:"keepUnsealed"`
	AnnounceToIPNI       bool          `json:"announceToIpni"`
	StartDelay           time.Duration `json:"startDelay"`
	Duration             time.Duration `json:"duration"`
	State                ScheduleState `json:"state"`
	ScheduleCron         string        `json:"scheduleCron"`
	ScheduleDealNumber   int           `json:"scheduleDealNumber"`
	ScheduleDealSize     int64         `json:"scheduleDealSize"`
	MaxPendingDealNumber int           `json:"maxPendingDealNumber"`
	MaxPendingDealSize   int64         `json:"maxPendingDealSize"`
	Notes                string        `json:"notes"`
	ErrorMessage         string        `json:"errorMessage"`
	AllowedPieceCIDs     StringSlice   `json:"allowedPieceCids"`
}

type ScheduleState 

type ScheduleState string
const (
	ScheduleActive    ScheduleState = "active"
	SchedulePaused    ScheduleState = "paused"
	ScheduleError     ScheduleState = "error"
	ScheduleCompleted ScheduleState = "completed"
)

type Source 

type Source struct {
	ID                   uint32     `gorm:"primaryKey"                                                             json:"id"`
	CreatedAt            time.Time  `json:"createdAt"`
	UpdatedAt            time.Time  `json:"updatedAt"`
	DatasetID            uint32     `gorm:"uniqueIndex:dataset_type_path"                                          json:"datasetId"`
	Dataset              *Dataset   `` /* 132-byte string literal not displayed */
	Type                 SourceType `gorm:"uniqueIndex:dataset_type_path;size:64"                                  json:"type"`
	Path                 string     `gorm:"uniqueIndex:dataset_type_path;size:1024"                                json:"path"`
	Metadata             Metadata   `gorm:"type:JSON"                                                              json:"metadata"`
	ScanIntervalSeconds  uint64     `json:"scanIntervalSeconds"`
	ScanningState        WorkState  `gorm:"index:source_cleanup;size:16"                                           json:"scanningState"`
	ScanningWorkerID     *string    `gorm:"index:source_cleanup;size:64"                                           json:"scanningWorkerId,omitempty"`
	ScanningWorker       *Worker    `` /* 132-byte string literal not displayed */
	LastScannedTimestamp int64      `json:"lastScannedTimestamp"`
	LastScannedPath      string     `json:"lastScannedPath"`
	ErrorMessage         string     `json:"errorMessage"`
	DeleteAfterExport    bool       `json:"deleteAfterExport"`
	DagGenState          WorkState  `json:"dagGenState"`
	DagGenWorkerID       *string    `gorm:"index;size:64"                                                          json:"dagGenWorkerId,omitempty"`
	DagGenWorker         *Worker    `` /* 132-byte string literal not displayed */
	DagGenErrorMessage   string     `json:"dagGenErrorMessage"`
	// contains filtered or unexported fields
}

Source represents a source of data, i.e. a local file system directory.

func (*Source) LoadRootDirectory added in v0.1.0 

func (s *Source) LoadRootDirectory(db *gorm.DB) error

func (*Source) RootDirectory 

func (s *Source) RootDirectory() *Directory

func (*Source) RootDirectoryID 

func (s *Source) RootDirectoryID(db *gorm.DB) (uint64, error)

type SourceType 

type SourceType = string
const (
	Local  SourceType = "local"
	Upload SourceType = "upload"
)

type StringSlice 

type StringSlice []string

func (*StringSlice) Scan 

func (ss *StringSlice) Scan(src any) error

func (StringSlice) Value 

func (ss StringSlice) Value() (driver.Value, error)

type Wallet 

type Wallet struct {
	ID         string `gorm:"primaryKey;size:16"   json:"id"`      // ID is the short ID of the wallet
	Address    string `gorm:"unique;size:256"      json:"address"` // Address is the Filecoin full address of the wallet
	PrivateKey string `json:"privateKey,omitempty"`                // PrivateKey is the private key of the wallet
	RemotePeer string `json:"remotePeer,omitempty"`                // RemotePeer is the remote peer ID of the wallet, for remote signing purpose
}

type WalletAssignment 

type WalletAssignment struct {
	ID        uint32   `gorm:"primaryKey"                                       json:"id"`
	WalletID  string   `json:"walletId"`
	Wallet    *Wallet  `gorm:"foreignKey:WalletID;constraint:OnDelete:CASCADE"  json:"wallet,omitempty"  swaggerignore:"true"`
	DatasetID uint32   `json:"datasetId"`
	Dataset   *Dataset `gorm:"foreignKey:DatasetID;constraint:OnDelete:CASCADE" json:"dataset,omitempty" swaggerignore:"true"`
}

type WorkState 

type WorkState string
const (
	// Created means the item has been created is not ready for processing.
	Created WorkState = ""
	// Ready means the item is ready to be processed.
	Ready WorkState = "ready"
	// Processing means the work is currently being processed.
	Processing WorkState = "processing"
	// Complete means the work is complete.
	Complete WorkState = "complete"
	// Error means the work has some error.
	Error WorkState = "error"
)

type WorkType 

type WorkType string
const (
	Scan         WorkType = "scan"
	DealMaking   WorkType = "deal_making"
	DealTracking WorkType = "deal_tracking"
	Packing      WorkType = "packing"
)

type Worker 

type Worker struct {
	ID            string    `gorm:"primaryKey;size:64" json:"id"`
	WorkType      WorkType  `json:"workType"`
	WorkingOn     string    `json:"workingOn"`
	LastHeartbeat time.Time `json:"lastHeartbeat"`
	Hostname      string    `json:"hostname"`
}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.