indexer

package
v0.5.7 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: May 27, 2022 License: Apache-2.0 Imports: 20 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	ValidatorAggCreatorTaskName      = "ValidatorAggCreator"
	ValidatorGroupAggCreatorTaskName = "ValidatorGroupAggCreator"
	ProposalAggCreatorTaskName       = "ProposalAggCreator"
)
View Source 
const (
	TargetIndexBlockSequences = iota + 1
	TargetIndexValidatorSequences
	TargetIndexValidatorGroupSequences
	TargetIndexValidatorAggregates
	TargetIndexValidatorGroupAggregates
)
View Source 
const (
	TaskNameHeightMetaDownloader      = "HeightMetaDownloader"
	TaskNameBlockDownloader           = "BlockDownloader"
	TaskNameValidatorsDownloader      = "ValidatorsDownloader"
	TaskNameValidatorGroupsDownloader = "ValidatorGroupsDownloader"
	TaskNameTransactionsDownloader    = "TransactionsDownloader"
)
View Source 
const (
	TaskNameHeightMetaFetcher      = "HeightMetaFetcher"
	TaskNameBlockFetcher           = "BlockFetcher"
	TaskNameValidatorsFetcher      = "ValidatorsFetcher"
	TaskNameValidatorGroupsFetcher = "ValidatorGroupsFetcher"
	TaskNameTransactionsFetcher    = "TransactionsFetcher"
)
View Source 
const (
	SyncerPersistorTaskName                = "SyncerPersistor"
	BlockSeqPersistorTaskName              = "BlockSeqPersistor"
	ValidatorSeqPersistorTaskName          = "ValidatorSeqPersistor"
	ValidatorGroupSeqPersistorTaskName     = "ValidatorGroupSeqPersistor"
	AccountActivitySeqPersistorTaskName    = "AccountActivitySeqPersistor"
	ValidatorAggPersistorTaskName          = "ValidatorAggPersistor"
	ValidatorGroupAggPersistorTaskName     = "ValidatorGroupAggPersistor"
	ProposalAggPersistorTaskName           = "ProposalAggPersistor"
	TaskNameSystemEventPersistor           = "SystemEventPersistor"
	GovernanceActivitySeqPersistorTaskName = "GovernanceActivitySeqPersistor"
)
View Source 
const (
	BlockSeqCreatorTaskName              = "BlockSeqCreator"
	ValidatorSeqCreatorTaskName          = "ValidatorSeqCreator"
	ValidatorGroupSeqCreatorTaskName     = "ValidatorGroupSeqCreator"
	AccountActivitySeqCreatorTaskName    = "AccountActivitySeqCreator"
	GovernanceActivitySeqCreatorTaskName = "GovernanceActivitySeqCreator"
)
View Source 
const (
	TaskNameHeightMetaUploader      = "HeightMetaUploader"
	TaskNameBlockUploader           = "BlockUploader"
	TaskNameValidatorsUploader      = "ValidatorsUploader"
	TaskNameValidatorGroupsUploader = "ValidatorGroupsUploader"
	TaskNameTransactionsUploader    = "TransactionsUploader"
)
View Source 
const (
	CtxReport = "context_report"
)
View Source 
const (
	MainSyncerTaskName = "MainSyncer"
)
View Source 
const (
	TaskNameGovernanceLogsParser = "GovernanceLogsParser"
)
View Source 
const (
	TaskNameSetup = "Setup"
)
View Source 
const (
	TaskNameSystemEventCreator = "SystemEventCreator"
)

Variables

View Source 
var (
	ErrGroupRewardOutsideOfRange = errors.New("group reward is outside of specified buckets")

	MaxValidatorSequences int64 = 1000
	MissedForMaxThreshold int64 = 50
	MissedInRowThreshold  int64 = 50
)
View Source 
var (
	OperationTypeInternalTransferReceived                     = fmt.Sprintf("%sReceived", figmentclient.OperationTypeInternalTransfer)
	OperationTypeInternalTransferSent                         = fmt.Sprintf("%sSent", figmentclient.OperationTypeInternalTransfer)
	OperationTypeValidatorGroupVoteCastReceived               = fmt.Sprintf("%sReceived", figmentclient.OperationTypeValidatorGroupVoteCast)
	OperationTypeValidatorGroupVoteCastSent                   = fmt.Sprintf("%sSent", figmentclient.OperationTypeValidatorGroupVoteCast)
	OperationTypeValidatorGroupVoteActivatedReceived          = fmt.Sprintf("%sReceived", figmentclient.OperationTypeValidatorGroupVoteActivated)
	OperationTypeValidatorGroupVoteActivatedSent              = fmt.Sprintf("%sSent", figmentclient.OperationTypeValidatorGroupVoteActivated)
	OperationTypeValidatorGroupPendingVoteRevokedReceived     = fmt.Sprintf("%sReceived", figmentclient.OperationTypeValidatorGroupPendingVoteRevoked)
	OperationTypeValidatorGroupPendingVoteRevokedSent         = fmt.Sprintf("%sSent", figmentclient.OperationTypeValidatorGroupPendingVoteRevoked)
	OperationTypeValidatorGroupActiveVoteRevokedReceived      = fmt.Sprintf("%sReceived", figmentclient.OperationTypeValidatorGroupActiveVoteRevoked)
	OperationTypeValidatorGroupActiveVoteRevokedSent          = fmt.Sprintf("%sSent", figmentclient.OperationTypeValidatorGroupActiveVoteRevoked)
	OperationTypeValidatorEpochPaymentDistributedForGroup     = fmt.Sprintf("%sForGroup", figmentclient.OperationTypeValidatorEpochPaymentDistributed)
	OperationTypeValidatorEpochPaymentDistributedForValidator = fmt.Sprintf("%sForValidator", figmentclient.OperationTypeValidatorEpochPaymentDistributed)

	ErrBlockSequenceNotValid          = errors.New("block sequence not valid")
	ErrValidatorSequenceNotValid      = errors.New("validator sequence not valid")
	ErrValidatorGroupSequenceNotValid = errors.New("validator group sequence not valid")
)
View Source 
var (
	StageAnalyzer pipeline.StageName = "stage_analyzer"

	ErrIsPristine          = errors.New("cannot run because database is empty")
	ErrIndexCannotBeRun    = errors.New("cannot run index process")
	ErrBackfillCannotBeRun = errors.New("cannot run backfill process")
)
View Source 
var (
	ErrNothingToProcess = errors.New("nothing to process")
)

Functions

func NewAccountActivitySeqCreatorTask added in v0.1.0 

func NewAccountActivitySeqCreatorTask(cfg *config.Config) *accountActivitySeqCreatorTask

NewAccountActivitySeqCreatorTask creates account activity sequences

func NewAccountActivitySeqPersistorTask added in v0.1.0 

func NewAccountActivitySeqPersistorTask(accountActivitySeqDb store.AccountActivitySeq) pipeline.Task

NewAccountActivitySeqPersistorTask is responsible for storing validator info to persistence layer

func NewBackfillSource 

func NewBackfillSource(cfg *config.Config, client figmentclient.Client, syncableDb store.Syncables, indexVersion int64) (*backfillSource, error)

func NewBlockDownloaderTask added in v0.2.0 

func NewBlockDownloaderTask() pipeline.Task

func NewBlockFetcherTask 

func NewBlockFetcherTask(client figmentclient.Client) pipeline.Task

func NewBlockSeqCreatorTask 

func NewBlockSeqCreatorTask(blockSeqDb store.BlockSeq) *blockSeqCreatorTask

NewBlockSeqCreatorTask creates block sequences

func NewBlockSeqPersistorTask 

func NewBlockSeqPersistorTask(blockSeqDb store.BlockSeq) pipeline.Task

NewBlockSeqPersistorTask is responsible for storing block to persistence layer

func NewBlockUploaderTask added in v0.2.0 

func NewBlockUploaderTask() pipeline.Task

func NewConfigParser 

func NewConfigParser(file string) (*configParser, error)

func NewGovernanceActivitySeqCreatorTask added in v0.1.0 

func NewGovernanceActivitySeqCreatorTask(cfg *config.Config) *governanceActivitySeqCreatorTask

NewGovernanceActivitySeqCreatorTask creates account activity sequences

func NewGovernanceActivitySeqPersistorTask added in v0.1.0 

func NewGovernanceActivitySeqPersistorTask(governanceActivitySeqDb store.GovernanceActivitySeq) pipeline.Task

NewGovernanceActivitySeqPersistorTask is responsible for storing validator info to persistence layer

func NewGovernanceLogsParserTask added in v0.1.0 

func NewGovernanceLogsParserTask() *governanceLogsParserTask

NewGovernanceLogsParserTask parses transaction logs to data about governance

func NewHeightMetaDownloaderTask added in v0.2.0 

func NewHeightMetaDownloaderTask() pipeline.Task

func NewHeightMetaFetcherTask added in v0.2.0 

func NewHeightMetaFetcherTask(client figmentclient.Client) pipeline.Task

func NewHeightMetaUploaderTask added in v0.2.0 

func NewHeightMetaUploaderTask() pipeline.Task

func NewIndexSource 

func NewIndexSource(cfg *config.Config, client figmentclient.Client, syncableDb store.Syncables, jobsDb store.Jobs, sourceCfg *IndexSourceConfig) (*indexSource, error)

func NewLogger 

func NewLogger() pipeline.Logger

func NewMainSyncerTask 

func NewMainSyncerTask(syncableDb store.Syncables) pipeline.Task

func NewPayloadFactory 

func NewPayloadFactory(dl *datalake.DataLake) *payloadFactory

func NewPipeline 

func NewPipeline(
	cfg *config.Config,
	client figmentclient.Client,
	dl *datalake.DataLake,

	syncableDb store.Syncables,
	jobsDb store.Jobs,
	databaseDb store.Database,
	reportsDb store.Reports,
	blockSeqDb store.BlockSeq,
	validatorSeqDb store.ValidatorSeq,
	accountActivitySeqDb store.AccountActivitySeq,
	validatorGroupSeqDb store.ValidatorGroupSeq,
	validatorAggDb store.ValidatorAgg,
	validatorGroupAggDb store.ValidatorGroupAgg,
	proposalAggDb store.ProposalAgg,
	systemEventDb store.SystemEvents,
	governanceActivitySeqDb store.GovernanceActivitySeq,
) (*indexingPipeline, error)

func NewProposalAggCreatorTask added in v0.1.0 

func NewProposalAggCreatorTask(proposalAggDb store.ProposalAgg) *proposalAggCreatorTask

func NewProposalAggPersistorTask added in v0.1.0 

func NewProposalAggPersistorTask(proposalAggDb store.ProposalAgg) pipeline.Task

NewProposalAggPersistorTask psql validator aggregate to persistence layer

func NewSetupTask added in v0.2.0 

func NewSetupTask(c figmentclient.Client) *setupTask

func NewSink 

func NewSink(syncableDb store.Syncables, databaseDb store.Database, c figmentclient.Client, versionNumber int64) *sink

func NewSyncerPersistorTask 

func NewSyncerPersistorTask(syncableDb store.Syncables) pipeline.Task

NewSyncerPersistorTask is responsible for storing syncable to persistence layer

func NewSystemEventCreatorTask added in v0.1.0 

func NewSystemEventCreatorTask(cfg *config.Config, validatorSeqDb store.ValidatorSeq, accountActivitySeqDb store.AccountActivitySeq, validatorGroupSeqDb store.ValidatorGroupSeq) *systemEventCreatorTask

NewSystemEventCreatorTask creates system events

func NewSystemEventPersistorTask added in v0.1.0 

func NewSystemEventPersistorTask(systemEventDb store.SystemEvents) pipeline.Task

NewSystemEventPersistorTask psql system events to persistance layer

func NewTransactionDownloaderTask added in v0.2.0 

func NewTransactionDownloaderTask() pipeline.Task

func NewTransactionFetcherTask added in v0.1.0 

func NewTransactionFetcherTask(client figmentclient.Client) pipeline.Task

func NewTransactionUploaderTask added in v0.2.0 

func NewTransactionUploaderTask() pipeline.Task

func NewValidatorAggCreatorTask 

func NewValidatorAggCreatorTask(validatorAggDb store.ValidatorAgg) *validatorAggCreatorTask

func NewValidatorAggPersistorTask 

func NewValidatorAggPersistorTask(validatorAggDb store.ValidatorAgg) pipeline.Task

NewValidatorAggPersistorTask store validator aggregate to persistence layer

func NewValidatorDownloaderTask added in v0.2.0 

func NewValidatorDownloaderTask() pipeline.Task

func NewValidatorFetcherTask 

func NewValidatorFetcherTask(client figmentclient.Client) pipeline.Task

func NewValidatorGroupAggCreatorTask 

func NewValidatorGroupAggCreatorTask(validatorGroupAggDb store.ValidatorGroupAgg) *validatorGroupAggCreatorTask

func NewValidatorGroupAggPersistorTask 

func NewValidatorGroupAggPersistorTask(validatorGroupAggDb store.ValidatorGroupAgg) pipeline.Task

NewValidatorGroupAggPersistorTask psql validator group aggregate to persistence layer

func NewValidatorGroupDownloaderTask added in v0.2.0 

func NewValidatorGroupDownloaderTask() pipeline.Task

func NewValidatorGroupFetcherTask 

func NewValidatorGroupFetcherTask(client figmentclient.Client) pipeline.Task

func NewValidatorGroupSeqCreatorTask 

func NewValidatorGroupSeqCreatorTask(cfg *config.Config) *validatorGroupSeqCreatorTask

NewValidatorGroupSeqCreatorTask creates validator era sequences

func NewValidatorGroupSeqPersistorTask 

func NewValidatorGroupSeqPersistorTask(validatorGroupSeqDb store.ValidatorGroupSeq) pipeline.Task

NewValidatorGroupSeqPersistorTask is responsible for storing validator era info to persistence layer

func NewValidatorGroupUploaderTask added in v0.2.0 

func NewValidatorGroupUploaderTask() pipeline.Task

func NewValidatorSeqCreatorTask 

func NewValidatorSeqCreatorTask(cfg *config.Config) *validatorSeqCreatorTask

NewValidatorSeqCreatorTask creates validator sequences

func NewValidatorSeqPersistorTask 

func NewValidatorSeqPersistorTask(validatorSeqDb store.ValidatorSeq) pipeline.Task

NewValidatorSeqPersistorTask is responsible for storing validator info to persistence layer

func NewValidatorUploaderTask added in v0.2.0 

func NewValidatorUploaderTask() pipeline.Task

func RunFetcherPipeline added in v0.2.0 

func RunFetcherPipeline(height int64, client figmentclient.Client, dl *datalake.DataLake) error

func ToAccountActivitySequence added in v0.1.0 

func ToAccountActivitySequence(syncable *model.Syncable, rawTransactions []*figmentclient.Transaction) ([]model.AccountActivitySeq, error)

func ToBlockSequence 

func ToBlockSequence(syncable *model.Syncable, rawBlock *figmentclient.Block) (*model.BlockSeq, error)

func ToGovernanceActivitySequence added in v0.1.0 

func ToGovernanceActivitySequence(syncable *model.Syncable, parsedGovernanceLogs []*ParsedGovernanceLogs) ([]model.GovernanceActivitySeq, error)

func ToValidatorGroupSequence 

func ToValidatorGroupSequence(syncable *model.Syncable, rawValidatorGroups []*figmentclient.ValidatorGroup, rawValidators []*figmentclient.Validator) ([]model.ValidatorGroupSeq, error)

func ToValidatorSequence 

func ToValidatorSequence(syncable *model.Syncable, rawValidators []*figmentclient.Validator) ([]model.ValidatorSeq, error)

Types

type BackfillConfig 

type BackfillConfig struct {
	Parallel  bool
	Force     bool
	TargetIds []int64
}

type BlockDownloaderTask added in v0.2.0 

type BlockDownloaderTask struct{}

func (*BlockDownloaderTask) GetName added in v0.2.0 

func (t *BlockDownloaderTask) GetName() string

func (*BlockDownloaderTask) Run added in v0.2.0 

func (t *BlockDownloaderTask) Run(ctx context.Context, p pipeline.Payload) error

type BlockFetcherTask 

type BlockFetcherTask struct {
	// contains filtered or unexported fields
}

func (*BlockFetcherTask) GetName 

func (t *BlockFetcherTask) GetName() string

func (*BlockFetcherTask) Run 

func (t *BlockFetcherTask) Run(ctx context.Context, p pipeline.Payload) error

type BlockUploaderTask added in v0.2.0 

type BlockUploaderTask struct{}

func (*BlockUploaderTask) GetName added in v0.2.0 

func (t *BlockUploaderTask) GetName() string

func (*BlockUploaderTask) Run added in v0.2.0 

func (t *BlockUploaderTask) Run(ctx context.Context, p pipeline.Payload) error

type ConfigParser 

type ConfigParser interface {
	GetCurrentVersionId() int64
	GetAllVersionedVersionIds() []int64
	IsAnyVersionSequential(versionIds []int64) bool
	GetAllAvailableTasks() []pipeline.TaskName
	GetAllVersionedTasks() ([]pipeline.TaskName, error)
	GetTasksByVersionIds([]int64) ([]pipeline.TaskName, error)
	GetTasksByTargetIds([]int64) ([]pipeline.TaskName, error)
}

type HeightMeta 

type HeightMeta struct {
	ChainId     uint64
	Height      int64
	Time        *types.Time
	Epoch       *int64
	EpochSize   *int64
	LastInEpoch *bool
}

type HeightMetaDownloaderTask added in v0.2.0 

type HeightMetaDownloaderTask struct{}

func (*HeightMetaDownloaderTask) GetName added in v0.2.0 

func (t *HeightMetaDownloaderTask) GetName() string

func (*HeightMetaDownloaderTask) Run added in v0.2.0 

func (t *HeightMetaDownloaderTask) Run(ctx context.Context, p pipeline.Payload) error

type HeightMetaFetcherTask added in v0.2.0 

type HeightMetaFetcherTask struct {
	// contains filtered or unexported fields
}

func (*HeightMetaFetcherTask) GetName added in v0.2.0 

func (t *HeightMetaFetcherTask) GetName() string

func (*HeightMetaFetcherTask) Run added in v0.2.0 

func (t *HeightMetaFetcherTask) Run(ctx context.Context, p pipeline.Payload) error

type HeightMetaUploaderTask added in v0.2.0 

type HeightMetaUploaderTask struct{}

func (*HeightMetaUploaderTask) GetName added in v0.2.0 

func (t *HeightMetaUploaderTask) GetName() string

func (*HeightMetaUploaderTask) Run added in v0.2.0 

func (t *HeightMetaUploaderTask) Run(ctx context.Context, p pipeline.Payload) error

type IndexConfig 

type IndexConfig struct {
	BatchSize   int64
	StartHeight int64
}

type IndexSourceConfig 

type IndexSourceConfig struct {
	BatchSize   int64
	StartHeight int64
}

type ParsedGovernanceLogs added in v0.1.0 

type ParsedGovernanceLogs struct {
	ProposalId      uint64
	Account         string
	TransactionHash string
	Kind            string
	Details         interface{}
}

type PipelineSyncableStore 

type PipelineSyncableStore interface {
	FindSmallestIndexVersion() (*int64, error)
}

type ProposalBackfill added in v0.3.9 

type ProposalBackfill struct {
	// contains filtered or unexported fields
}

func NewProposalBackfill added in v0.3.9 

func NewProposalBackfill(c theceloclient.Client, proposalAggDb store.ProposalAgg, syncables store.Syncables, dl *datalake.DataLake) *ProposalBackfill

func (ProposalBackfill) Run added in v0.3.9 

func (p ProposalBackfill) Run(ctx context.Context, startBlock, endBlock int64, startProposalId, endProposalId int) error

type RunConfig 

type RunConfig struct {
	Height            int64
	DesiredVersionIDs []int64
	DesiredTargetIDs  []int64
	Dry               bool
}

type TransactionsDownloaderTask added in v0.2.0 

type TransactionsDownloaderTask struct{}

func (*TransactionsDownloaderTask) GetName added in v0.2.0 

func (t *TransactionsDownloaderTask) GetName() string

func (*TransactionsDownloaderTask) Run added in v0.2.0 

func (t *TransactionsDownloaderTask) Run(ctx context.Context, p pipeline.Payload) error

type TransactionsFetcherTask added in v0.1.0 

type TransactionsFetcherTask struct {
	// contains filtered or unexported fields
}

func (*TransactionsFetcherTask) GetName added in v0.1.0 

func (t *TransactionsFetcherTask) GetName() string

func (*TransactionsFetcherTask) Run added in v0.1.0 

func (t *TransactionsFetcherTask) Run(ctx context.Context, p pipeline.Payload) error

type TransactionsUploaderTask added in v0.2.0 

type TransactionsUploaderTask struct{}

func (*TransactionsUploaderTask) GetName added in v0.2.0 

func (t *TransactionsUploaderTask) GetName() string

func (*TransactionsUploaderTask) Run added in v0.2.0 

func (t *TransactionsUploaderTask) Run(ctx context.Context, p pipeline.Payload) error

type ValidatorGroupsDownloaderTask added in v0.2.0 

type ValidatorGroupsDownloaderTask struct{}

func (*ValidatorGroupsDownloaderTask) GetName added in v0.2.0 

func (t *ValidatorGroupsDownloaderTask) GetName() string

func (*ValidatorGroupsDownloaderTask) Run added in v0.2.0 

func (t *ValidatorGroupsDownloaderTask) Run(ctx context.Context, p pipeline.Payload) error

type ValidatorGroupsFetcherTask 

type ValidatorGroupsFetcherTask struct {
	// contains filtered or unexported fields
}

func (*ValidatorGroupsFetcherTask) GetName 

func (t *ValidatorGroupsFetcherTask) GetName() string

func (*ValidatorGroupsFetcherTask) Run 

func (t *ValidatorGroupsFetcherTask) Run(ctx context.Context, p pipeline.Payload) error

type ValidatorGroupsUploaderTask added in v0.2.0 

type ValidatorGroupsUploaderTask struct{}

func (*ValidatorGroupsUploaderTask) GetName added in v0.2.0 

func (t *ValidatorGroupsUploaderTask) GetName() string

func (*ValidatorGroupsUploaderTask) Run added in v0.2.0 

func (t *ValidatorGroupsUploaderTask) Run(ctx context.Context, p pipeline.Payload) error

type ValidatorsDownloaderTask added in v0.2.0 

type ValidatorsDownloaderTask struct{}

func (*ValidatorsDownloaderTask) GetName added in v0.2.0 

func (t *ValidatorsDownloaderTask) GetName() string

func (*ValidatorsDownloaderTask) Run added in v0.2.0 

func (t *ValidatorsDownloaderTask) Run(ctx context.Context, p pipeline.Payload) error

type ValidatorsFetcherTask 

type ValidatorsFetcherTask struct {
	// contains filtered or unexported fields
}

func (*ValidatorsFetcherTask) GetName 

func (t *ValidatorsFetcherTask) GetName() string

func (*ValidatorsFetcherTask) Run 

func (t *ValidatorsFetcherTask) Run(ctx context.Context, p pipeline.Payload) error

type ValidatorsUploaderTask added in v0.2.0 

type ValidatorsUploaderTask struct{}

func (*ValidatorsUploaderTask) GetName added in v0.2.0 

func (t *ValidatorsUploaderTask) GetName() string

func (*ValidatorsUploaderTask) Run added in v0.2.0 

func (t *ValidatorsUploaderTask) Run(ctx context.Context, p pipeline.Payload) error

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.