db

package
v0.0.0-...-bbd84dc Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Nov 5, 2024 License: Apache-2.0 Imports: 42 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

View Source 
var (
	ErrEmptyAgentVersion = fmt.Errorf("empty agent version")
	ErrEmptyProtocol     = fmt.Errorf("empty protocol")
	ErrEmptyProtocolsSet = fmt.Errorf("empty protocols set")
)
View Source 
var ErrorStr = map[string]string{}
View Source 
var KnownErrors = map[string]string{
	"i/o timeout":                                models.NetErrorIoTimeout,
	"RPC timeout":                                models.NetErrorIoTimeout,
	"no recent network activity":                 models.NetErrorIoTimeout,
	"handshake did not complete in time":         models.NetErrorIoTimeout,
	"connection refused":                         models.NetErrorConnectionRefused,
	"connection reset by peer":                   models.NetErrorConnectionResetByPeer,
	"protocol not supported":                     models.NetErrorProtocolNotSupported,
	"protocols not supported":                    models.NetErrorProtocolNotSupported,
	"peer id mismatch":                           models.NetErrorPeerIDMismatch,
	"peer IDs don't match":                       models.NetErrorPeerIDMismatch,
	"no route to host":                           models.NetErrorNoRouteToHost,
	"network is unreachable":                     models.NetErrorNetworkUnreachable,
	"no good addresses":                          models.NetErrorNoGoodAddresses,
	"context deadline exceeded":                  models.NetErrorIoTimeout,
	"no public IP address":                       models.NetErrorNoIPAddress,
	"max dial attempts exceeded":                 models.NetErrorMaxDialAttemptsExceeded,
	"host is down":                               models.NetErrorHostIsDown,
	"stream reset":                               models.NetErrorStreamReset,
	"stream closed":                              models.NetErrorStreamReset,
	"failed to negotiate security protocol: EOF": models.NetErrorNegotiateSecurityProtocol,
	"failed to negotiate stream multiplexer":     models.NetErrorNegotiateStreamMultiplexer,
	"resource limit exceeded":                    models.NetErrorResourceLimitExceeded,
	"Write on stream":                            models.NetErrorWriteOnStream,
	"can't assign requested address":             models.NetErrorCantAssignRequestedAddress,
	"cannot assign requested address":            models.NetErrorCantAssignRequestedAddress,
	"connection gated":                           models.NetErrorConnectionGated,
	"RESOURCE_LIMIT_EXCEEDED (201)":              models.NetErrorCantConnectOverRelay,
	"NO_RESERVATION (204)":                       models.NetErrorCantConnectOverRelay,

	"no good ip address":                models.NetErrorNoIPAddress,
	"disconnect requested":              models.NetErrorDevp2pDisconnectRequested,
	"network error":                     models.NetErrorDevp2pNetworkError,
	"breach of protocol":                models.NetErrorDevp2pBreachOfProtocol,
	"useless peer":                      models.NetErrorDevp2pUselessPeer,
	"too many peers":                    models.NetErrorDevp2pTooManyPeers,
	"already connected":                 models.NetErrorDevp2pAlreadyConnected,
	"incompatible p2p protocol version": models.NetErrorDevp2pIncompatibleP2PProtocolVersion,
	"invalid node identity":             models.NetErrorDevp2pInvalidNodeIdentity,
	"client quitting":                   models.NetErrorDevp2pClientQuitting,
	"unexpected identity":               models.NetErrorDevp2pUnexpectedIdentity,
	"connected to self":                 models.NetErrorDevp2pConnectedToSelf,
	"read timeout":                      models.NetErrorDevp2pReadTimeout,
	"subprotocol error":                 models.NetErrorDevp2pSubprotocolError,
	"could not negotiate eth protocol":  models.NetErrorDevp2pEthprotocolError,
	"handshake failed: EOF":             models.NetErrorDevp2pHandshakeEOF,
	"malformed disconnect message":      models.NetErrorDevp2pMalformedDisconnectMessage,
}

KnownErrors contains a list of known errors. Property key + string to match for

Functions

func NetError 

func NetError(err error) string

NetError extracts the appropriate error type from the given error.

func Rollback 

func Rollback(txn *sql.Tx)

Rollback calls rollback on the given transaction and logs the potential error.

Types

type Client 

type Client interface {
	io.Closer
	InitCrawl(ctx context.Context, version string) (*models.Crawl, error)
	UpdateCrawl(ctx context.Context, crawl *models.Crawl) error
	QueryBootstrapPeers(ctx context.Context, limit int) ([]peer.AddrInfo, error)
	PersistCrawlProperties(ctx context.Context, crawl *models.Crawl, properties map[string]map[string]int) error
	PersistCrawlVisit(ctx context.Context, crawlID int, peerID peer.ID, maddrs []ma.Multiaddr, protocols []string, agentVersion string, connectDuration time.Duration, crawlDuration time.Duration, visitStartedAt time.Time, visitEndedAt time.Time, connectErrorStr string, crawlErrorStr string, properties null.JSON) (*InsertVisitResult, error)
	PersistNeighbors(ctx context.Context, crawl *models.Crawl, dbPeerID *int, peerID peer.ID, errorBits uint16, dbNeighborsIDs []int, neighbors []peer.ID) error
}

func InitJSONClient 

func InitJSONClient(out string) (Client, error)

InitJSONClient .

func NewClient 

func NewClient(ctx context.Context, cfg *config.Database) (Client, error)

NewClient will initialize the right database client based on the given configuration. This can either be a Postgres, JSON, or noop client. The noop client is a dummy implementation of the Client interface that does nothing when the methods are called. That's the one used if the user specifies `--dry-run` on the command line. The JSON client is used when the user specifies a JSON output directory. Then JSON files with crawl information are written to that directory. In any other case, the Postgres client is used.

type DBClient 

type DBClient struct {
	// contains filtered or unexported fields
}

func InitDBClient 

func InitDBClient(ctx context.Context, cfg *config.Database) (*DBClient, error)

InitDBClient establishes a database connection with the provided configuration and applies any pending migrations.

func (*DBClient) Close 

func (c *DBClient) Close() error

func (*DBClient) FetchDueOpenSessions 

func (c *DBClient) FetchDueOpenSessions(ctx context.Context) (models.SessionsOpenSlice, error)

FetchDueOpenSessions fetches all open sessions from the database that are due.

func (*DBClient) FetchUnresolvedMultiAddresses 

func (c *DBClient) FetchUnresolvedMultiAddresses(ctx context.Context, limit int) (models.MultiAddressSlice, error)

FetchUnresolvedMultiAddresses fetches all multi addresses that were not resolved yet.

func (*DBClient) GetOrCreateAgentVersionID 

func (c *DBClient) GetOrCreateAgentVersionID(ctx context.Context, exec boil.ContextExecutor, agentVersion string) (*int, error)

func (*DBClient) GetOrCreateProtocol 

func (c *DBClient) GetOrCreateProtocol(ctx context.Context, exec boil.ContextExecutor, protocol string) (*int, error)

func (*DBClient) GetOrCreateProtocolsSetID 

func (c *DBClient) GetOrCreateProtocolsSetID(ctx context.Context, exec boil.ContextExecutor, protocols []string) (*int, error)

func (*DBClient) Handle 

func (c *DBClient) Handle() *sql.DB

func (*DBClient) InitCrawl 

func (c *DBClient) InitCrawl(ctx context.Context, version string) (*models.Crawl, error)

InitCrawl inserts a crawl instance into the database in the state `started`. This is done to receive a database ID that all subsequent database entities can be linked to.

func (*DBClient) PersistCrawlProperties 

func (c *DBClient) PersistCrawlProperties(ctx context.Context, crawl *models.Crawl, properties map[string]map[string]int) error

func (*DBClient) PersistCrawlVisit 

func (c *DBClient) PersistCrawlVisit(
	ctx context.Context,
	crawlID int,
	peerID peer.ID,
	maddrs []ma.Multiaddr,
	protocols []string,
	agentVersion string,
	connectDuration time.Duration,
	crawlDuration time.Duration,
	visitStartedAt time.Time,
	visitEndedAt time.Time,
	connectErrorStr string,
	crawlErrorStr string,
	properties null.JSON,
) (*InsertVisitResult, error)

func (*DBClient) PersistDialVisit 

func (c *DBClient) PersistDialVisit(
	ctx context.Context,
	peerID peer.ID,
	maddrs []ma.Multiaddr,
	dialDuration time.Duration,
	visitStartedAt time.Time,
	visitEndedAt time.Time,
	errorStr string,
) (*InsertVisitResult, error)

func (*DBClient) PersistNeighbors 

func (c *DBClient) PersistNeighbors(ctx context.Context, crawl *models.Crawl, dbPeerID *int, peerID peer.ID, errorBits uint16, dbNeighborsIDs []int, neighbors []peer.ID) error

func (*DBClient) QueryBootstrapPeers 

func (c *DBClient) QueryBootstrapPeers(ctx context.Context, limit int) ([]peer.AddrInfo, error)

func (*DBClient) UpdateCrawl 

func (c *DBClient) UpdateCrawl(ctx context.Context, crawl *models.Crawl) error

UpdateCrawl takes the crawl model an updates it in the database.

func (*DBClient) UpsertPeer 

func (c *DBClient) UpsertPeer(mh string, agentVersionID null.Int, protocolSetID null.Int, properties null.JSON) (int, error)

type InsertVisitResult 

type InsertVisitResult struct {
	PID       peer.ID
	PeerID    *int
	VisitID   *int
	SessionID *int
}

func (*InsertVisitResult) Scan 

func (ivr *InsertVisitResult) Scan(value interface{}) error

type JSONClient 

type JSONClient struct {
	// contains filtered or unexported fields
}

func (*JSONClient) Close 

func (n *JSONClient) Close() error

func (*JSONClient) InitCrawl 

func (c *JSONClient) InitCrawl(ctx context.Context, version string) (*models.Crawl, error)

func (*JSONClient) PersistCrawlProperties 

func (c *JSONClient) PersistCrawlProperties(ctx context.Context, crawl *models.Crawl, properties map[string]map[string]int) error

func (*JSONClient) PersistCrawlVisit 

func (c *JSONClient) PersistCrawlVisit(ctx context.Context, crawlID int, peerID peer.ID, maddrs []ma.Multiaddr, protocols []string, agentVersion string, connectDuration time.Duration, crawlDuration time.Duration, visitStartedAt time.Time, visitEndedAt time.Time, connectErrorStr string, crawlErrorStr string, properties null.JSON) (*InsertVisitResult, error)

func (*JSONClient) PersistNeighbors 

func (c *JSONClient) PersistNeighbors(ctx context.Context, crawl *models.Crawl, dbPeerID *int, peerID peer.ID, errorBits uint16, dbNeighborsIDs []int, neighbors []peer.ID) error

func (*JSONClient) QueryBootstrapPeers 

func (c *JSONClient) QueryBootstrapPeers(ctx context.Context, limit int) ([]peer.AddrInfo, error)

func (*JSONClient) UpdateCrawl 

func (c *JSONClient) UpdateCrawl(ctx context.Context, crawl *models.Crawl) error

type JSONNeighbors 

type JSONNeighbors struct {
	PeerID      peer.ID
	NeighborIDs []peer.ID
	ErrorBits   string
}

type JSONVisit 

type JSONVisit struct {
	PeerID          peer.ID
	Maddrs          []ma.Multiaddr
	Protocols       []string
	AgentVersion    string
	ConnectDuration string
	CrawlDuration   string
	VisitStartedAt  time.Time
	VisitEndedAt    time.Time
	ConnectErrorStr string
	CrawlErrorStr   string
	Properties      null.JSON
}

type NoopClient 

type NoopClient struct{}

func InitNoopClient 

func InitNoopClient() *NoopClient

func (*NoopClient) Close 

func (n *NoopClient) Close() error

func (*NoopClient) InitCrawl 

func (n *NoopClient) InitCrawl(ctx context.Context, version string) (*models.Crawl, error)

func (*NoopClient) PersistCrawlProperties 

func (n *NoopClient) PersistCrawlProperties(ctx context.Context, crawl *models.Crawl, properties map[string]map[string]int) error

func (*NoopClient) PersistCrawlVisit 

func (n *NoopClient) PersistCrawlVisit(ctx context.Context, crawlID int, peerID peer.ID, maddrs []ma.Multiaddr, protocols []string, agentVersion string, connectDuration time.Duration, crawlDuration time.Duration, visitStartedAt time.Time, visitEndedAt time.Time, connectErrorStr string, crawlErrorStr string, properties null.JSON) (*InsertVisitResult, error)

func (*NoopClient) PersistNeighbors 

func (n *NoopClient) PersistNeighbors(ctx context.Context, crawl *models.Crawl, dbPeerID *int, peerID peer.ID, errorBits uint16, dbNeighborsIDs []int, neighbors []peer.ID) error

func (*NoopClient) QueryBootstrapPeers 

func (n *NoopClient) QueryBootstrapPeers(ctx context.Context, limit int) ([]peer.AddrInfo, error)

func (*NoopClient) UpdateCrawl 

func (n *NoopClient) UpdateCrawl(ctx context.Context, crawl *models.Crawl) error

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.