cmd

Documentation

Index

• Variables
• func CalculateCommp(t *testing.T, content []byte, targetPieceSize uint64) string
• func CompareDirectories(t *testing.T, dir1, dir2 string)
• func Download(ctx context.Context, url string, nThreads int) ([]byte, error)
• func GetAllPieceCIDs(content string) []string
• func GetFirstCID(content string) string
• func SetVersionJSON(versionJSON []byte) error
• func SetupErrorHandler()
• func SetupHelpPager()
• func WaitForServerReady(ctx context.Context, url string) error
• type Runner
  • func NewRunner() *Runner
  • func (r *Runner) Run(ctx context.Context, args string) (string, string, error)
  • func (r *Runner) Save(t *testing.T, tempDirs ...string)
  • func (r *Runner) WithMode(mode RunnerMode) *Runner
• type RunnerMode

Variables

var App = &cli.App{
	Name:  "singularity",
	Usage: "A tool for large-scale clients with PB-scale data onboarding to Filecoin network",
	Description: `Database Backend Support:
  Singularity supports multiple database backend: sqlite3, postgres, mysql5.7+
  Use '--database-connection-string' or $DATABASE_CONNECTION_STRING to specify the database connection string.
    Example for postgres  - postgres://user:pass@example.com:5432/dbname
    Example for mysql     - mysql://user:pass@tcp(localhost:3306)/dbname?parseTime=true
    Example for sqlite3   - sqlite:/absolute/path/to/database.db
                or        - sqlite:relative/path/to/database.db

Network Support:
  Default settings in Singularity are for Mainnet. You may set below environment values for other network:
    For Calibration network:
      * Set LOTUS_API to https://api.calibration.node.glif.io/rpc/v1
      * Set MARKET_DEAL_URL to https://marketdeals-calibration.s3.amazonaws.com/StateMarketDeals.json.zst
      * Set LOTUS_TEST to 1
    For all other networks:
      * Set LOTUS_API to your network's Lotus API endpoint
      * Set MARKET_DEAL_URL to empty string
      * Set LOTUS_TEST to 0 or 1 based on whether the network address starts with 'f' or 't'
    Switching between different networks with the same database instance is not recommended.

Logging:
  Singularity uses go-log for logging and can be controlled by below environment variables:
    * GOLOG_LOG_LEVEL  - example values: debug, info, warn, error, dpanic, panic, fatal
    * GOLOG_LOG_FMT    - example values: color, nocolor, json
    * More details can be found at https://github.com/ipfs/go-log

Upgrading:
  Within each minor version upgrade, use "singularity admin init" to upgrade the database schema.
  For major version upgrade, please refer to the release notes for upgrade instructions.
`,
	EnableBashCompletion: true,
	Flags: []cli.Flag{
		&cli.StringFlag{
			Name:        "database-connection-string",
			Usage:       "Connection string to the database",
			DefaultText: "sqlite:" + "./singularity.db",
			Value:       "sqlite:" + "./singularity.db",
			EnvVars:     []string{"DATABASE_CONNECTION_STRING"},
		},
		&cli.BoolFlag{
			Name:  "json",
			Usage: "Enable JSON output",
			Value: false,
		},
		&cli.BoolFlag{
			Name:  "verbose",
			Usage: "Enable verbose output. This will print more columns for the result as well as full error trace",
			Value: false,
		},
		&cli.StringFlag{
			Name:     "lotus-api",
			Category: "Lotus",
			Usage:    "Lotus RPC API endpoint",
			Value:    "https://api.node.glif.io/rpc/v1",
			EnvVars:  []string{"LOTUS_API"},
		},
		&cli.StringFlag{
			Name:     "lotus-token",
			Category: "Lotus",
			Usage:    "Lotus RPC API token",
			Value:    "",
			EnvVars:  []string{"LOTUS_TOKEN"},
		},
		&cli.BoolFlag{
			Name:     "lotus-test",
			Category: "Lotus",
			Usage:    "Whether the runtime environment is using Testnet.",
			EnvVars:  []string{"LOTUS_TEST"},
		},
	},
	Before: func(c *cli.Context) error {
		if c.Bool("lotus-test") {
			address.CurrentNetwork = address.Testnet
			logger.Infow("Current network is set to Testnet")
		} else {
			address.CurrentNetwork = address.Mainnet
		}
		return nil
	},
	Commands: []*cli.Command{
		ez.PrepCmd,
		VersionCmd,
		{
			Name:     "admin",
			Usage:    "Admin commands",
			Category: "Operations",
			Subcommands: []*cli.Command{
				admin.InitCmd,
				admin.ResetCmd,
				admin.MigrateDatasetCmd,
				admin.MigrateScheduleCmd,
			},
		},
		DownloadCmd,
		tool.ExtractCarCmd,
		{
			Name:     "deal",
			Usage:    "Replication / Deal making management",
			Category: "Operations",
			Subcommands: []*cli.Command{
				{
					Name:  "schedule",
					Usage: "Schedule deals",
					Subcommands: []*cli.Command{
						schedule.CreateCmd,
						schedule.ListCmd,
						schedule.UpdateCmd,
						schedule.PauseCmd,
						schedule.ResumeCmd,
						schedule.RemoveCmd,
					},
				},
				deal.SendManualCmd,
				deal.ListCmd,
			},
		},
		{
			Name:     "run",
			Category: "Daemons",
			Usage:    "run different singularity components",
			Subcommands: []*cli.Command{
				run.APICmd,
				run.DatasetWorkerCmd,
				run.ContentProviderCmd,
				run.DealTrackerCmd,
				run.DealPusherCmd,
				run.DownloadServerCmd,
			},
		},
		{
			Name:     "wallet",
			Category: "Operations",
			Usage:    "Wallet management",
			Subcommands: []*cli.Command{
				wallet.ImportCmd,
				wallet.ListCmd,
				wallet.RemoveCmd,
			},
		},
		{
			Name:     "storage",
			Category: "Operations",
			Usage:    "Create and manage storage system connections",
			Subcommands: []*cli.Command{
				storage.CreateCmd,
				storage.ExploreCmd,
				storage.ListCmd,
				storage.RemoveCmd,
				storage.UpdateCmd,
				storage.RenameCmd,
			},
		},
		{
			Name:     "prep",
			Category: "Operations",
			Usage:    "Create and manage dataset preparations",
			Subcommands: []*cli.Command{
				dataprep.CreateCmd,
				dataprep.ListCmd,
				dataprep.StatusCmd,
				dataprep.RenameCmd,
				dataprep.AttachSourceCmd,
				dataprep.AttachOutputCmd,
				dataprep.DetachOutputCmd,
				dataprep.StartScanCmd,
				dataprep.PauseScanCmd,
				dataprep.StartPackCmd,
				dataprep.PausePackCmd,
				dataprep.StartDagGenCmd,
				dataprep.PauseDagGenCmd,
				dataprep.ListPiecesCmd,
				dataprep.AddPieceCmd,
				dataprep.ExploreCmd,
				dataprep.AttachWalletCmd,
				dataprep.ListWalletsCmd,
				dataprep.DetachWalletCmd,
				dataprep.RemoveCmd,
			},
		},
	},
}

var DownloadCmd = &cli.Command{
	Name:      "download",
	Usage:     "Download a CAR file from the metadata API",
	Category:  "Utility",
	Before:    cliutil.CheckNArgs,
	ArgsUsage: "<piece_cid>",
	Flags: func() []cli.Flag {
		flags := []cli.Flag{
			&cli.StringFlag{
				Name:     "api",
				Usage:    "URL of the metadata API",
				Value:    "http://127.0.0.1:7777",
				Category: "General Config",
			},
			&cli.StringFlag{
				Name:     "out-dir",
				Usage:    "Directory to write CAR files to",
				Value:    ".",
				Category: "General Config",
			},
			&cli.IntFlag{
				Name:     "concurrency",
				Usage:    "Number of concurrent downloads",
				Value:    10,
				Category: "General Config",
			},
			&cli.BoolFlag{
				Name:     "quiet",
				Usage:    "Suppress all output",
				Category: "General Config",
				Value:    false,
			},
		}

		flags = append(flags, storage.HTTPClientConfigFlagsForUpdate...)
		flags = append(flags, storage.CommonConfigFlags...)

		keys := make(map[string]struct{})
		for _, backend := range storagesystem.Backends {
			for _, providerOptions := range backend.ProviderOptions {
				for _, option := range providerOptions.Options {
					if !model.IsSecretConfigName(option.Name) {
						continue
					}
					flag := option.ToCLIFlag(backend.Prefix+"-", false, backend.Description)
					if _, ok := keys[flag.Names()[0]]; ok {
						continue
					}
					keys[flag.Names()[0]] = struct{}{}
					flags = append(flags, flag)
				}
			}
		}
		return flags
	}(),
	Action: func(c *cli.Context) error {
		api := c.String("api")
		outDir := c.String("out-dir")
		concurrency := c.Int("concurrency")
		piece := c.Args().First()
		config := map[string]string{}
		for _, key := range c.LocalFlagNames() {
			if c.IsSet(key) {
				if slices.Contains([]string{"api", "out-dir", "concurrency", "quiet"}, key) {
					continue
				}
				value := c.String(key)
				config[key] = value
			}
		}
		clientConfig, err := storage.GetClientConfigForUpdate(c)
		if err != nil {
			return errors.WithStack(err)
		}
		err = handler.DownloadHandler(c, piece, api, config, *clientConfig, outDir, concurrency)
		if err == nil {
			log.Logger("Download").Info("Download complete")
			return nil
		}
		return errors.WithStack(err)
	},
}

var Version string

var VersionCmd = &cli.Command{
	Name:    "version",
	Usage:   "Print version information",
	Aliases: []string{"v"},
	Action: func(context *cli.Context) error {
		buildInfo, ok := debug.ReadBuildInfo()
		if !ok {
			fmt.Println("unknown version")
		}

		version := buildInfo.Main.Version
		if version == "(devel)" || version == "" {
			version = Version
		}
		var revision string
		var modified string
		for _, setting := range buildInfo.Settings {
			switch setting.Key {
			case "vcs.revision":
				revision = setting.Value[:7]
			case "vcs.modified":
				modified = setting.Value
			}
		}
		if revision == "" {
			revision = "-unknown"
		} else {
			revision = "-" + revision
		}
		switch modified {
		case "true":
			modified = "-dirty"
		case "false":
			modified = ""
		case "":
			modified = "-unknown"
		default:
			modified = "-" + modified
		}
		v := fmt.Sprintf("singularity %s%s%s\n", version, revision, modified)
		_, err := context.App.Writer.Write([]byte(v))

		return errors.WithStack(err)
	},
}

Functions

func CalculateCommp

func CalculateCommp(t *testing.T, content []byte, targetPieceSize uint64) string

func CompareDirectories

func CompareDirectories(t *testing.T, dir1, dir2 string)

func Download

func Download(ctx context.Context, url string, nThreads int) ([]byte, error)

func GetAllPieceCIDs

func GetAllPieceCIDs(content string) []string

func GetFirstCID

func GetFirstCID(content string) string

func SetVersionJSON

func SetVersionJSON(versionJSON []byte) error

func SetupErrorHandler

func SetupErrorHandler()

func SetupHelpPager

func SetupHelpPager()

func WaitForServerReady

func WaitForServerReady(ctx context.Context, url string) error

Types

type Runner

type Runner struct {
	// contains filtered or unexported fields
}

func NewRunner

func NewRunner() *Runner

NewRunner creates a new Runner to capture CLI args

Note: tests invoking this function should stay in cmd.Test package because this function relies on environment variables to set database connection string so it won't work with parallel execution of different test packages.

func (*Runner) Run

func (r *Runner) Run(ctx context.Context, args string) (string, string, error)

func (*Runner) Save

func (r *Runner) Save(t *testing.T, tempDirs ...string)

func (*Runner) WithMode

func (r *Runner) WithMode(mode RunnerMode) *Runner

type RunnerMode

type RunnerMode string

const (
	Normal  RunnerMode = "normal"
	Verbose RunnerMode = "verbose"
	JSON    RunnerMode = "json"
)