whypfs

README

WhyPFS Core

A lightweight importable library to run a WhyPFS node.

Easiest way to setup an WhyPFS / Estuary core node that peers with any nodes within the IPFS network. This core library decouples the node initialization setup based on the Why's https://github.com/whyrusleeping/whypfs

Features

• Significantly improved default Bitswap parameters
• Default usage of the accelerated DHT client
• Default flatfs parameters work well at large scales on good filesystems (ext4, xfs, probably others)

Setup a new WhyPFS node

Using the default will give the most optimal configuration based on whyPFS. It'll also use the default boostrap nodes.

go get github.com/application-research/whypfs-core

Setup a node

peer, err := NewNode(NewNodeParams{Ctx: context.Background()})	
if err != nil {
    t.Fatal(err)
}

Set up a node with your own config

// initialize a node parameter
params := NewNodeParams{
    Ctx:       context.Background(),
    Datastore: NewInMemoryDatastore(),
}

// create a new config of your own
newConfig := &Config{
    Offline:           true,
    ReprovideInterval: 0,
    Libp2pKeyFile:     "mykey",
    ListenAddrs:       []string{"/ip4/127.0.0.1/tcp/0"},
    AnnounceAddrs:     nil,
    DatastoreDir: struct {
        Directory string
        Options   leveldb.Options
    }{},
    Blockstore:              "",
    NoBlockstoreCache:       false,
    NoAnnounceContent:       false,
    NoLimiter:               false,
    BitswapConfig:           BitswapConfig{},
    ConnectionManagerConfig: ConnectionManager{},
}

// set it
params.Config = params.ConfigurationBuilder(newConfig)
myNode, err := NewNode(params)
if err1 != nil {
    t.Fatal(err)
}

Add/Pin and Get a file

node, err := peer.AddPinFile(context.Background(), bytes.NewReader([]byte("letsrebuildtolearnnewthings!")), nil)
content, err := peer.GetFile(context.Background(), node.Cid())

Add/Pin Directory and Get the ipld.Node of the directory

node, err := peer.AddPinDirectory(context.Background(), "./test/test_directory")
mainDirNode, err := peer.GetDirectory(context.Background(), node)

Provides

• An ipld.DAGService.
• AddPinFile method to add a file to a node
• AddPinDirectory function to add a directory to a node
• GetFile function to get a file using a CID
• GetDirectory function to retrieve an entire directory from a ipld.Node
• Custom bootstrap nodes

Examples

There are a few examples on how to utilize the node which includes

• how to create a running peer node.
• how to create a CAR file and add it to the peer node
• how to add a file / dir to the peer node.

Documentation

Index

• Variables
• func DefaultBootstrapPeers() []peer.AddrInfo
• func NewInMemoryDatastore() datastore.Batching
• func NewResourceManager() (network.ResourceManager, error)
• type AddParams
• type BitswapConfig
• type Config
  • func SetConfigDefaults() *Config
• type ConnectionManager
• type DeleteManyBlockstore
• type NewNodeParams
  • func (n NewNodeParams) ConfigurationBuilder(config *Config) *Config
• type Node
  • func NewNode(nodeParams NewNodeParams) (*Node, error)
  • func (p *Node) AddPinDirectory(ctx context.Context, path string) (ipld.Node, error)
  • func (p *Node) AddPinFile(ctx context.Context, r io.Reader, params *AddParams) (ipld.Node, error)
  • func (p *Node) BlockStore() blockstore.Blockstore
  • func (p *Node) BootstrapPeers(peers []peer.AddrInfo)
  • func (p *Node) GetDirectory(ctx context.Context, c ipld.Node) (ufsio.Directory, error)
  • func (p *Node) GetDirectoryWithCid(ctx context.Context, c cid.Cid) (ufsio.Directory, error)
  • func (p *Node) GetFile(ctx context.Context, c cid.Cid) (ufsio.ReadSeekCloser, error)
  • func (p *Node) HasBlock(ctx context.Context, c cid.Cid) (bool, error)
  • func (p *Node) Session(ctx context.Context) ipld.NodeGetter

Variables

var BootstrapPeers []peer.AddrInfo

var Libp2pOptionsExtra = []libp2p.Option{
	libp2p.NATPortMap(),
	libp2p.ConnectionManager(connMgr),
	libp2p.EnableAutoRelay(),
	libp2p.EnableNATService(),
}

Libp2pOptionsExtra provides some useful libp2p options to create a fully featured libp2p Host. It can be used with SetupLibp2p.

Functions

func DefaultBootstrapPeers

func DefaultBootstrapPeers() []peer.AddrInfo

Creating a list of multiaddresses that are used to bootstrap the network.

func NewInMemoryDatastore

func NewInMemoryDatastore() datastore.Batching

NewInMemoryDatastore provides a sync Datastore that lives in-memory only and is not persisted.

func NewResourceManager

func NewResourceManager() (network.ResourceManager, error)

Types

type AddParams

type AddParams struct {
	Layout    string
	Chunker   string
	RawLeaves bool
	Hidden    bool
	Shard     bool
	NoCopy    bool
	HashFun   string
}

AddParams contains all of the configurable parameters needed to specify the importing process of a file.

type BitswapConfig

type BitswapConfig struct {
	MaxOutstandingBytesPerPeer int64
	TargetMessageSize          int
}

type Config

type Config struct {
	// The DAGService will not announce or retrieve blocks from the network
	Offline bool
	// ReprovideInterval sets how often to reprovide records to the DHT
	ReprovideInterval time.Duration
	Libp2pKeyFile     string
	ListenAddrs       []string
	AnnounceAddrs     []string
	DatastoreDir      struct {
		Directory string
		Options   levelds.Options
	}
	Blockstore              string
	NoBlockstoreCache       bool
	NoAnnounceContent       bool
	NoLimiter               bool
	BitswapConfig           BitswapConfig
	Limits                  rcmgr.ScalingLimitConfig
	ConnectionManagerConfig ConnectionManager
}

func SetConfigDefaults

func SetConfigDefaults() *Config

type ConnectionManager

type ConnectionManager struct {
	HighWater int
	LowWater  int
}

type DeleteManyBlockstore

type DeleteManyBlockstore interface {
	blockstore.Blockstore
	DeleteMany(context.Context, []cid.Cid) error
}

type NewNodeParams

type NewNodeParams struct {
	// Context is the context to use for the node.
	Ctx        context.Context
	Repo       string
	Datastore  datastore.Batching
	Blockstore blockstore.Blockstore
	Dht        *dht.IpfsDHT
	Config     *Config
}

func (NewNodeParams) ConfigurationBuilder

func (n NewNodeParams) ConfigurationBuilder(config *Config) *Config

type Node

type Node struct {
	//	node context
	Ctx context.Context

	//	Node configuration
	Config *Config

	// hosts
	Host       host.Host
	Dht        *dht.IpfsDHT
	StorageDir string

	// dag service
	ipld.DAGService
	Blockstore   blockstore.Blockstore
	Blockservice blockservice.BlockService
	Datastore    datastore.Batching
	Reprovider   provider.System
	Exchange     exchange.Interface
	Bitswap      *bitswap.Bitswap
	FilDht       *dht.IpfsDHT
	FullRt       *fullrt.FullRT
}

func NewNode

func NewNode(nodeParams NewNodeParams) (*Node, error)

NewNode creates a new WhyPFS node with the given configuration.

func (*Node) AddPinDirectory

func (p *Node) AddPinDirectory(ctx context.Context, path string) (ipld.Node, error)

Adding the directory of the pin to the path.

func (*Node) AddPinFile

func (p *Node) AddPinFile(ctx context.Context, r io.Reader, params *AddParams) (ipld.Node, error)

AddPinFile chunks and adds content to the DAGService from a reader. The content is stored as a UnixFS DAG (default for IPFS). It returns the root ipld.Node.

func (*Node) BlockStore

func (p *Node) BlockStore() blockstore.Blockstore

BlockStore offers access to the Blockstore underlying the Peer's DAGService.

func (*Node) BootstrapPeers

func (p *Node) BootstrapPeers(peers []peer.AddrInfo)

Helper function to bootstrap peers

Bootstrapping the node to the network.

func (*Node) GetDirectory

func (p *Node) GetDirectory(ctx context.Context, c ipld.Node) (ufsio.Directory, error)

Getting the directory from the node.

func (*Node) GetDirectoryWithCid

func (p *Node) GetDirectoryWithCid(ctx context.Context, c cid.Cid) (ufsio.Directory, error)

Getting the directory with the cid.

func (*Node) GetFile

func (p *Node) GetFile(ctx context.Context, c cid.Cid) (ufsio.ReadSeekCloser, error)

GetFile returns a reader to a file as identified by its root CID. The file must have been added as a UnixFS DAG (default for IPFS).

func (*Node) HasBlock

func (p *Node) HasBlock(ctx context.Context, c cid.Cid) (bool, error)

HasBlock returns whether a given block is available locally. It is a shorthand for .Blockstore().Has().

func (*Node) Session

func (p *Node) Session(ctx context.Context) ipld.NodeGetter

Creating a new function called Session that takes in a context and returns a NodeGetter.