jiffy

README

⚡ Jiffy

Just In-Time Filecoin For You

Jiffy provides a seamless interface to store and retrieve arbitrary data blobs on the Filecoin network. It offers an efficient data preparation system, ensuring data is scanned just once before being prepped for a Filecoin deal. Each uploaded blob in Jiffy is identified with a unique piece CID, which can be leveraged for merkle inclusion proofs. Additionally, Jiffy's bin-packing mechanism optimizes storage by fitting blobs into full Filecoin sectors and ensures data integrity by checking both the storage provider and the Filecoin chain.

Its API is designed to be fully compatible with Motion and includes an integrated Motion blob store.

Features

• Arbitrary Blob Management: Easily store io.Reader data on Filecoin and retrieve it as io.ReadSeekCloser.
• One-Pass Data Preparation: Processes input data in a single pass for Filecoin storage.
• Unique Piece CID for Blobs: Every stored blob gets its own Piece CID for use in merkle inclusion proofs.
• Optimized Bin Packing: Efficiently packs blobs across sectors to ensure cost-effective storage on Filecoin.
• Customizable Replication: Decide the storage providers and the replication factor for each piece.
• Integrated with Motion Blob Store: Contains a built-in Motion blob store implementation.
• Efficient Byte-Range Retrieval (WIP): Retrieves data with Boost Piece CID range request and performs on-the-fly content verification.
• HTTP Server Offloading (WIP): Routes data to a local HTTP server, authenticated using per deal proposal JWT.
• S3-Compatible Offloading (WIP): Allows data routing to any S3-compatible API.

Development Status

🚧 Under Active Development 🚧

We aim to keep the main branch stable. However, as this is an evolving project, breaking changes might occur. For production environments, please rely on released versions and always consult the release notes before upgrading.

Documentation

For an in-depth understanding and integration specifics, refer to the godoc documentation.

License

This project is dual-licensed under the MIT and Apache 2.0 licenses. For more details, consult the LICENSE.md file.

Documentation

Index

• Constants
• Variables
• type Dealer
• type Jiffy
  • func New(o ...Option) (*Jiffy, error)
  • func (j *Jiffy) GetReplicas(ctx context.Context, info abi.PieceInfo) ([]Replica, error)
  • func (j *Jiffy) GetSegment(ctx context.Context, info abi.PieceInfo) (*Segment, error)
  • func (j *Jiffy) ListSegments(ctx context.Context) ([]*Segment, error)
  • func (j *Jiffy) Retrieve(ctx context.Context, info abi.PieceInfo) (io.ReadSeekCloser, error)
  • func (j *Jiffy) Segment(ctx context.Context, closer io.ReadCloser) (*Segment, error)
  • func (j *Jiffy) Shutdown(ctx context.Context) error
  • func (j *Jiffy) Start(ctx context.Context) error
• type Offload
• type Offloader
• type Option
• type Piece
  • func NewPiece(capacity abi.PaddedPieceSize) *Piece
• type Replica
  • func (r *Replica) Expiration(genesis time.Time) time.Time
  • func (r *Replica) MatchesProposalOnChain() bool
  • func (r *Replica) PieceCID() cid.Cid
  • func (r *Replica) Provider() address.Address
  • func (r *Replica) Status(head abi.ChainEpoch) ReplicaStatus
• type ReplicaStatus
  • func (rs ReplicaStatus) String() string
• type Replicator
• type Retriever
• type Segment
• type Segmentor
• type Segments
  • func (s Segments) Len() int
  • func (s Segments) Less(i, j int) bool
  • func (s Segments) Swap(i, j int)
• type Wallet

Constants

const (
	KiB = 1 << 10
	MiB = KiB * KiB
	GiB = KiB * MiB
)

Variables

var (
	// ErrSegmentTooLarge signals that segment data exceeds the maximum allowed.
	ErrSegmentTooLarge = errors.New("segment data exceeds the maximum allowed")

	//ErrSegmentNotFound signals that the segment corresponding to a given piece CID is not found.
	ErrSegmentNotFound = errors.New("segment not found")
)

Types

type Dealer

type Dealer interface {
	Deal(context.Context, *Piece, address.Address) (*boostly.DealProposal, error)
}

type Jiffy

type Jiffy struct {
	// contains filtered or unexported fields
}

func New

func New(o ...Option) (*Jiffy, error)

func (*Jiffy) GetReplicas

func (j *Jiffy) GetReplicas(ctx context.Context, info abi.PieceInfo) ([]Replica, error)

func (*Jiffy) GetSegment

func (j *Jiffy) GetSegment(ctx context.Context, info abi.PieceInfo) (*Segment, error)

func (*Jiffy) ListSegments

func (j *Jiffy) ListSegments(ctx context.Context) ([]*Segment, error)

func (*Jiffy) Retrieve

func (j *Jiffy) Retrieve(ctx context.Context, info abi.PieceInfo) (io.ReadSeekCloser, error)

func (*Jiffy) Segment

func (j *Jiffy) Segment(ctx context.Context, closer io.ReadCloser) (*Segment, error)

func (*Jiffy) Shutdown

func (j *Jiffy) Shutdown(ctx context.Context) error

func (*Jiffy) Start

func (j *Jiffy) Start(ctx context.Context) error

type Offload

type Offload struct {
	Type    string
	URL     *url.URL
	Headers map[string]string
}

type Offloader

type Offloader interface {
	Offload(*Piece) (*Offload, error)
}

type Option

type Option func(*options) error

Option represents a configurable parameter in Motion service.

type Piece

type Piece struct {
	Info     abi.PieceInfo
	Capacity abi.PaddedPieceSize
	Segments Segments
	// TotalSegmentedSize is the sum of all Segment.SegmentedSize segments in this piece.
	TotalSegmentedSize uint64
}

func NewPiece

func NewPiece(capacity abi.PaddedPieceSize) *Piece

type Replica

type Replica struct {
	DealProposal       boostly.DealProposal
	LastProviderStatus *boostly.DealStatusResponse
	LastChainStatus    *telefil.StateMarketStorageDeal
	LastChecked        time.Time
	LastError          error
}

func (*Replica) Expiration

func (r *Replica) Expiration(genesis time.Time) time.Time

func (*Replica) MatchesProposalOnChain

func (r *Replica) MatchesProposalOnChain() bool

func (*Replica) PieceCID

func (r *Replica) PieceCID() cid.Cid

func (*Replica) Provider

func (r *Replica) Provider() address.Address

func (*Replica) Status

func (r *Replica) Status(head abi.ChainEpoch) ReplicaStatus

type ReplicaStatus

type ReplicaStatus int

const (

	// Unknown signals that the replica status is not known due to error during verification. See Replica.LastError.
	Unknown ReplicaStatus = iota
	// Accepted signals that the deal proposal has been accepted by the provider and is in process of being executed.
	Accepted
	// Slashed signals that the deal corresponding to the replica has been slashed.
	Slashed
	// Expired signals that the deal corresponding to the replica has expired.
	Expired
	// Published signals that the deal corresponding to the replica has been published on chain.
	Published
	// Active signals that the deal corresponding to the replica is active.
	Active
)

func (ReplicaStatus) String

func (rs ReplicaStatus) String() string

type Replicator

type Replicator interface {
	GetReplicas(context.Context, abi.PieceInfo) ([]Replica, error)
}

type Retriever

type Retriever interface {
	Retrieve(context.Context, abi.PieceInfo) (io.ReadSeekCloser, error)
}

type Segment

type Segment struct {
	Info abi.PieceInfo
	// Size returns the original data size
	RawSize uint64
	// SegmentedSize is the CAR-ified size.
	SegmentedSize uint64
	// CreateTime is the time at which this segment was created.
	CreateTime time.Time
}

type Segmentor

type Segmentor interface {
	Segment(context.Context, io.ReadCloser) (*Segment, error)
	GetSegment(context.Context, abi.PieceInfo) (*Segment, error)
	ListSegments(context.Context) ([]*Segment, error)
}

type Segments

type Segments []*Segment

func (Segments) Len

func (s Segments) Len() int

func (Segments) Less

func (s Segments) Less(i, j int) bool

func (Segments) Swap

func (s Segments) Swap(i, j int)

type Wallet

type Wallet interface {
	Sign(context.Context, []byte) (*crypto.Signature, error)
	Address() (address.Address, error)
}