onion

README

Onion

Onion is a tool to test the various layers of the Decentralised GW(RHEA) stack both in isolation and together vis a vis ipfs.io to flush out correctness discrepancies across the RHEA stack wrt ipfs.io.

It does this by replaying traffic that confirms to the verifiable GW spec from a given logfile to the various layers of the RHEA stack and also to ipfs.io and then comparing the responses from the various layers wrt ipfs.io to flag correctness discrepancies.

These layers include:

1. Lassie
2. Saturn L1 Shim without Nginx + Lassie
3. Nginx + Saturn L1 shim + Lassie
4. Bifrost-GW + Saturn Nginx + L1 Shim + Lassie

How To

1. We use docker compose to bring up all the aforementioned layers/components for RHEA. Please clone the add_onion_tests branch of the https://github.com/filecoin-saturn/L1-node repo and follow the comprehensive instructions on https://github.com/filecoin-saturn/L1-node/tree/add_onion_tests/integration/onion for how to bring up the containers

   Always run `rm -rf ~/shared/onion_l1/nginx_cache` before 
   bringing up the containers to ensure a clean slate
   

2. Please ensure that the docker compose file you are using has the versions of the various components you want to test before bringing up the containers. Notes on how to update the version can also be found on the aforementioned instructions
3. Configure the config.toml file in the onion directory to point to the correct host and ports of all the layers

The default configuration should work fine unless you've made changes to the docker compose config

1. Run go build ./cmd/onion
2. Run ./onion -c={COUNT_OF_UNIQUE_REQUESTS} -f={LOG_FILE_TO_REPLAY} -n_runs=1 to run one round of an Onion test. This will replay requests from the log file to all layers of the RHEA stack and also to ipfs.io and publish a report wrt response code and response bytes correctness. It will also create multiple files/artefacts in the results directory that you can use to debug correctness discrepancies

Note on log files:

The log file should be a file with new line delimited URLs, one URL for each request. The URL should be a URL that satisfies the verifiable GW spec. Here's an example of a log file:

"https://{host:port}/ipfs/{cid}/metadata/2486?format=car&dag-scope=all&car-scope=all&depth=all"
"https://{host:port}/ipfs/{cid}/182?format=car&dag-scope=all&car-scope=all&depth=all"
"https://{host:port}/ipfs/{cid}/metadata/2486?format=car&dag-scope=all&car-scope=all&depth=all"
"https://{host:port}/ipfs/{cid}/182?format=car&dag-scope=all&car-scope=all&depth=all"

Documentation

Index

• func ExtractRaw(carBytes []byte) (response []byte, err error)
• func ParseCidFromPath(path string) string
• func PushMetrics(runID uuid.UUID) error
• type CidContactChecker
  • func NewCidContactChecker(mismatches []string) *CidContactChecker
  • func (klm *CidContactChecker) Check()
  • func (klm *CidContactChecker) GetCidContactResponse(cid string) (*CidContactOutput, error)
• type CidContactOutput
• type RequestExecutor
  • func NewRequestExecutor(reqs map[string]URLsToTest, n int, id uuid.UUID, dir string, rrdir string) *RequestExecutor
  • func (re *RequestExecutor) Execute()
  • func (re *RequestExecutor) WriteMismatchesToFile()
  • func (re *RequestExecutor) WriteResultsToFile()
• type ResponseBytesMismatch
• type Result
• type Results
• type URLBuilder
  • func NewURLBuilder(lassieIP, l1ShimIP, l1NginxIP, bifrostIP string) *URLBuilder
  • func (b *URLBuilder) BuildBifrostUrl(bifrostUrl string) string
  • func (b *URLBuilder) BuildKuboGWUrl(bifrostUrl string) string
  • func (b *URLBuilder) BuildL1NginxUrl(bifrostUrl string) string
  • func (b *URLBuilder) BuildL1ShimUrl(bifrostUrl string) string
  • func (b *URLBuilder) BuildLassieUrl(bifrostUrl string) string
  • func (ub *URLBuilder) BuildURLsToTest(bifrostReqUrl string) URLsToTest
• type URLsToTest

Functions

func ExtractRaw

func ExtractRaw(carBytes []byte) (response []byte, err error)

func ParseCidFromPath

func ParseCidFromPath(path string) string

func PushMetrics

func PushMetrics(runID uuid.UUID) error

Types

type CidContactChecker

type CidContactChecker struct {
	// contains filtered or unexported fields
}

func NewCidContactChecker

func NewCidContactChecker(mismatches []string) *CidContactChecker

func (*CidContactChecker) Check

func (klm *CidContactChecker) Check()

func (*CidContactChecker) GetCidContactResponse

func (klm *CidContactChecker) GetCidContactResponse(cid string) (*CidContactOutput, error)

type CidContactOutput

type CidContactOutput struct {
	Status     int
	Response   string
	IsDagHouse bool
	IsPinata   bool
}

type RequestExecutor

type RequestExecutor struct {
	// contains filtered or unexported fields
}

func NewRequestExecutor

func NewRequestExecutor(reqs map[string]URLsToTest, n int, id uuid.UUID, dir string, rrdir string) *RequestExecutor

func (*RequestExecutor) Execute

func (re *RequestExecutor) Execute()

func (*RequestExecutor) WriteMismatchesToFile

func (re *RequestExecutor) WriteMismatchesToFile()

func (*RequestExecutor) WriteResultsToFile

func (re *RequestExecutor) WriteResultsToFile()

type ResponseBytesMismatch

type ResponseBytesMismatch struct {
	KuboLassieMismatches    map[string]Results
	KuboLassieMismatchPaths []string

	KuboL1ShimMismatches    map[string]Results
	KuboL1ShimMismatchPaths []string

	KuboL1NginxMismatches    map[string]Results
	KuboL1NginxMismatchPaths []string

	KuboBifrostMismatches    map[string]Results
	KuboBifrostMismatchPaths []string

	LassieShimMismatches    map[string]Results
	LassieShimMismatchPaths []string

	ShimNginxMismatches    map[string]Results
	ShimNginxMismatchPaths []string

	NginxBifrostMismatches    map[string]Results
	NginxBifrostMismatchPaths []string

	TotalLassieReadSuccess  int
	TotalL1ShimReadSuccess  int
	TotalL1NginxReadSuccess int
	TotalBifrostReadSuccess int

	TotalLassieReadError  int
	TotalL1ShimReadError  int
	TotalL1NginxReadError int
	TotalBifrostReadError int

	LassieReadErrors  map[string]*Result
	L1ShimReadErrors  map[string]*Result
	L1NginxReadErrors map[string]*Result
	BifrostReadErrors map[string]*Result

	LassieReadErrorPaths  []string
	L1ShimReadErrorPaths  []string
	L1NginxReadErrorPaths []string
	BifrostReadErrorPaths []string

	TotalKuboLassieMatches   int
	TotalKuboL1ShimMatches   int
	TotalKuboL1NginxMatches  int
	TotalKuboBifrostMatches  int
	TotalNginxBifrostMatches int
}

type Result

type Result struct {
	Url        string
	StatusCode int
	Headers    map[string][]string
	ErrorBody  string

	ResponseBodyReadError string
	ResponseBody          []byte
	ResponseSize          uint64
}

type Results

type Results struct {
	KuboGWResult  *Result
	LassieResult  *Result
	L1ShimResult  *Result
	L1NginxResult *Result

	BifrostResult *Result
}

type URLBuilder

type URLBuilder struct {
	// contains filtered or unexported fields
}

func NewURLBuilder

func NewURLBuilder(lassieIP, l1ShimIP, l1NginxIP, bifrostIP string) *URLBuilder

func (*URLBuilder) BuildBifrostUrl

func (b *URLBuilder) BuildBifrostUrl(bifrostUrl string) string

func (*URLBuilder) BuildKuboGWUrl

func (b *URLBuilder) BuildKuboGWUrl(bifrostUrl string) string

func (*URLBuilder) BuildL1NginxUrl

func (b *URLBuilder) BuildL1NginxUrl(bifrostUrl string) string

func (*URLBuilder) BuildL1ShimUrl

func (b *URLBuilder) BuildL1ShimUrl(bifrostUrl string) string

func (*URLBuilder) BuildLassieUrl

func (b *URLBuilder) BuildLassieUrl(bifrostUrl string) string

func (*URLBuilder) BuildURLsToTest

func (ub *URLBuilder) BuildURLsToTest(bifrostReqUrl string) URLsToTest

type URLsToTest

type URLsToTest struct {
	Path string

	Lassie    string
	L1Shim    string
	L1Nginx   string
	KuboGWUrl string

	BifrostURL string
}