commands

Documentation

Index

• Variables

Variables

var CleanCommand = cli.Command{
	Name:        "clean",
	Usage:       "clean",
	Description: "Cleans up unused layers",

	Flags: []cli.Flag{
		cli.Int64Flag{
			Name:  "threshold-bytes",
			Usage: "Disk usage of the store directory at which cleanup should trigger",
		},
	},

	Action: func(ctx *cli.Context) error {
		logger := ctx.App.Metadata["logger"].(lager.Logger)
		logger = logger.Session("clean")
		newExitError := newErrorHandler(logger, "clean")

		configBuilder := ctx.App.Metadata["configBuilder"].(*config.Builder)
		configBuilder.WithCleanThresholdBytes(ctx.Int64("threshold-bytes"),
			ctx.IsSet("threshold-bytes"))

		cfg, err := configBuilder.Build()
		logger.Debug("clean-config", lager.Data{"currentConfig": cfg})
		if err != nil {
			logger.Error("config-builder-failed", err)
			return newExitError(err.Error(), 1)
		}

		storePath := cfg.StorePath
		if _, err = os.Stat(storePath); os.IsNotExist(err) {
			err = errorspkg.Errorf("no store found at %s", storePath)
			logger.Error("store-path-failed", err, nil)
			return newExitError(err.Error(), 0)
		}

		fsDriver, err := createFileSystemDriver(cfg)
		if err != nil {
			logger.Error("failed-to-initialise-filesystem-driver", err)
			return newExitError(err.Error(), 1)
		}

		imageCloner := imageClonerpkg.NewImageCloner(fsDriver, storePath)
		metricsEmitter := metrics.NewEmitter()

		locksmith := locksmithpkg.NewExclusiveFileSystem(storePath, metricsEmitter)
		dependencyManager := dependency_manager.NewDependencyManager(
			filepath.Join(storePath, storepkg.MetaDirName, "dependencies"),
		)

		storeNamespacer := groot.NewStoreNamespacer(storePath)
		idMappings, err := storeNamespacer.Read()
		if err != nil {
			logger.Error("reading-namespace-file", err)
			return newExitError(err.Error(), 1)
		}

		runner := linux_command_runner.New()
		idMapper := unpackerpkg.NewIDMapper(cfg.NewuidmapBin, cfg.NewgidmapBin, runner)
		nsFsDriver := namespaced.New(fsDriver, idMappings, idMapper, runner)
		sm := storepkg.NewStoreMeasurer(storePath, fsDriver)
		gc := garbage_collector.NewGC(nsFsDriver, imageCloner, dependencyManager)

		cleaner := groot.IamCleaner(locksmith, sm, gc, metricsEmitter)

		defer func() {
			unusedVols, err := gc.UnusedVolumes(logger, nil)
			if err != nil {
				logger.Error("getting-unused-layers-failed", err)
				return
			}
			metricsEmitter.TryEmitUsage(logger, "UnusedLayersSize", sm.CacheUsage(logger, unusedVols), "bytes")
		}()

		noop, err := cleaner.Clean(logger, cfg.Clean.ThresholdBytes, nil)
		if err != nil {
			logger.Error("cleaning-up-unused-resources", err)
			return newExitError(err.Error(), 1)
		}

		if noop {
			fmt.Println("threshold not reached: skipping clean")
			return nil
		}

		fmt.Println("clean completed")

		usage, err := sm.Usage(logger)
		if err != nil {
			logger.Error("measuring-store", err)
			return newExitError(err.Error(), 1)
		}

		metricsEmitter.TryIncrementRunCount("clean", nil)
		metricsEmitter.TryEmitUsage(logger, "StoreUsage", usage, "bytes")
		return nil
	},
}

var CreateCommand = cli.Command{
	Name:        "create",
	Usage:       "create [options] <image> <id>",
	Description: "Creates a root filesystem for the provided image.",

	Flags: []cli.Flag{
		cli.Int64Flag{
			Name:  "disk-limit-size-bytes",
			Usage: "Inclusive disk limit (i.e: includes all layers in the filesystem)",
		},
		cli.StringSliceFlag{
			Name:  "insecure-registry",
			Usage: "Whitelist a private registry",
		},
		cli.BoolFlag{
			Name:  "exclude-image-from-quota",
			Usage: "Set disk limit to be exclusive (i.e.: excluding image layers)",
		},
		cli.BoolFlag{
			Name:  "skip-layer-validation",
			Usage: "Do not validate checksums of image layers. (Can only be used with oci:/// protocol images.)",
		},
		cli.BoolFlag{
			Name:  "with-clean",
			Usage: "Clean up unused layers before creating rootfs",
		},
		cli.BoolFlag{
			Name:  "without-clean",
			Usage: "Do NOT clean up unused layers before creating rootfs",
		},
		cli.Int64Flag{
			Name:  "threshold-bytes",
			Usage: "Disk usage of the store directory at which cleanup should trigger",
		},
		cli.BoolFlag{
			Name:  "with-mount",
			Usage: "Mount the root filesystem after creation. This may require root privileges.",
		},
		cli.BoolFlag{
			Name:  "without-mount",
			Usage: "Do not mount the root filesystem.",
		},
		cli.StringFlag{
			Name:  "username",
			Usage: "Username to authenticate in image registry",
		},
		cli.StringFlag{
			Name:  "password",
			Usage: "Password to authenticate in image registry",
		},
	},

	Action: func(ctx *cli.Context) error {
		logger := ctx.App.Metadata["logger"].(lager.Logger)
		logger = logger.Session("create")
		newExitError := newErrorHandler(logger, "create")

		if ctx.NArg() != 2 {
			logger.Error("parsing-command", errorspkg.New("invalid arguments"), lager.Data{"args": ctx.Args()})
			return newExitError(fmt.Sprintf("invalid arguments - usage: %s", ctx.Command.Usage), 1)
		}

		configBuilder := ctx.App.Metadata["configBuilder"].(*config.Builder)
		configBuilder.WithInsecureRegistries(ctx.StringSlice("insecure-registry")).
			WithDiskLimitSizeBytes(ctx.Int64("disk-limit-size-bytes"),
				ctx.IsSet("disk-limit-size-bytes")).
			WithExcludeImageFromQuota(ctx.Bool("exclude-image-from-quota"),
				ctx.IsSet("exclude-image-from-quota")).
			WithSkipLayerValidation(ctx.Bool("skip-layer-validation"),
				ctx.IsSet("skip-layer-validation")).
			WithCleanThresholdBytes(ctx.Int64("threshold-bytes"), ctx.IsSet("threshold-bytes")).
			WithClean(ctx.IsSet("with-clean"), ctx.IsSet("without-clean")).
			WithMount(ctx.IsSet("with-mount"), ctx.IsSet("without-mount"))

		cfg, err := configBuilder.Build()
		logger.Debug("create-config", lager.Data{"currentConfig": cfg})
		if err != nil {
			logger.Error("config-builder-failed", err)
			return newExitError(err.Error(), 1)
		}

		if err = validateOptions(ctx, cfg); err != nil {
			return newExitError(err.Error(), 1)
		}

		storePath := cfg.StorePath
		id := ctx.Args().Tail()[0]
		baseImage := ctx.Args().First()
		baseImageURL, err := url.Parse(baseImage)
		if err != nil {
			logger.Error("base-image-url-parsing-failed", err)
			return newExitError(err.Error(), 1)
		}

		fsDriver, err := createFileSystemDriver(cfg)
		if err != nil {
			return newExitError(err.Error(), 1)
		}

		metricsEmitter := metrics.NewEmitter()
		sharedLocksmith := locksmithpkg.NewSharedFileSystem(storePath, metricsEmitter)
		exclusiveLocksmith := locksmithpkg.NewExclusiveFileSystem(storePath, metricsEmitter)
		imageCloner := image_cloner.NewImageCloner(fsDriver, storePath)

		storeNamespacer := groot.NewStoreNamespacer(storePath)
		manager := manager.New(storePath, storeNamespacer, fsDriver, fsDriver, fsDriver)
		if !manager.IsStoreInitialized(logger) {
			logger.Error("store-verification-failed", errors.New("store is not initialized"))
			return newExitError("Store path is not initialized. Please run init-store.", 1)
		}

		idMappings, err := storeNamespacer.Read()
		if err != nil {
			logger.Error("reading-namespace-file", err)
			return newExitError(err.Error(), 1)
		}

		runner := linux_command_runner.New()
		var unpacker base_image_puller.Unpacker
		unpackerStrategy := unpackerpkg.UnpackStrategy{
			Name:               "btrfs",
			WhiteoutDevicePath: filepath.Join(storePath, "whiteout_dev"),
		}

		var idMapper unpackerpkg.IDMapper
		if os.Getuid() == 0 {
			unpacker, err = unpackerpkg.NewTarUnpacker(unpackerStrategy)
			if err != nil {
				return newExitError(err.Error(), 1)
			}
		} else {
			idMapper = unpackerpkg.NewIDMapper(cfg.NewuidmapBin, cfg.NewgidmapBin, runner)
			unpacker = unpackerpkg.NewNSIdMapperUnpacker(runner, idMapper, unpackerStrategy)
		}

		dependencyManager := dependency_manager.NewDependencyManager(
			filepath.Join(storePath, storepkg.MetaDirName, "dependencies"),
		)

		nsFsDriver := namespaced.New(fsDriver, idMappings, idMapper, runner)

		systemContext := createSystemContext(baseImageURL, cfg.Create, ctx.String("username"), ctx.String("password"))

		baseImagePuller := base_image_puller.NewBaseImagePuller(
			createFetcher(baseImageURL, systemContext, cfg.Create),
			unpacker,
			nsFsDriver,
			metricsEmitter,
			exclusiveLocksmith,
		)

		sm := storepkg.NewStoreMeasurer(storePath, fsDriver)
		gc := garbage_collector.NewGC(nsFsDriver, imageCloner, dependencyManager)
		cleaner := groot.IamCleaner(exclusiveLocksmith, sm, gc, metricsEmitter)

		defer func() {
			unusedVols, err := gc.UnusedVolumes(logger, nil)
			if err != nil {
				logger.Error("getting-unused-layers-failed", err)
				return
			}
			metricsEmitter.TryEmitUsage(logger, "UnusedLayersSize", sm.CacheUsage(logger, unusedVols), "bytes")
		}()

		creator := groot.IamCreator(
			imageCloner, baseImagePuller, sharedLocksmith,
			dependencyManager, metricsEmitter, cleaner,
		)

		createSpec := groot.CreateSpec{
			ID:                          id,
			Mount:                       !cfg.Create.WithoutMount,
			BaseImageURL:                baseImageURL,
			DiskLimit:                   cfg.Create.DiskLimitSizeBytes,
			ExcludeBaseImageFromQuota:   cfg.Create.ExcludeImageFromQuota,
			UIDMappings:                 idMappings.UIDMappings,
			GIDMappings:                 idMappings.GIDMappings,
			CleanOnCreate:               cfg.Create.WithClean,
			CleanOnCreateThresholdBytes: cfg.Clean.ThresholdBytes,
		}
		image, err := creator.Create(logger, createSpec)
		if err != nil {
			logger.Error("creating", err)
			humanizedError := tryHumanize(err, createSpec)
			return newExitError(humanizedError, 1)
		}

		containerSpec := specs.Spec{
			Root: &specs.Root{
				Path: image.Rootfs,
			},
			Process: &specs.Process{
				Env: image.Image.Config.Env,
			},
			Mounts: []specs.Mount{},
		}

		for _, mount := range image.Mounts {
			containerSpec.Mounts = append(containerSpec.Mounts, specs.Mount{
				Destination: mount.Destination,
				Type:        mount.Type,
				Source:      mount.Source,
				Options:     mount.Options,
			})
		}

		jsonBytes, err := json.Marshal(containerSpec)
		if err != nil {
			logger.Error("formatting output", err)
			return newExitError(err.Error(), 1)
		}
		fmt.Println(string(jsonBytes))

		usage, err := sm.Usage(logger)
		if err != nil {
			logger.Error("measuring-store", err)
			return newExitError(err.Error(), 1)
		}

		metricsEmitter.TryIncrementRunCount("create", nil)
		metricsEmitter.TryEmitUsage(logger, "StoreUsage", usage, "bytes")

		return nil
	},
}

var DeleteCommand = cli.Command{
	Name:        "delete",
	Usage:       "delete <id|image path>",
	Description: "Deletes a container image",

	Action: func(ctx *cli.Context) error {
		logger := ctx.App.Metadata["logger"].(lager.Logger)
		logger = logger.Session("delete")
		newExitError := newErrorHandler(logger, "delete")

		if ctx.NArg() != 1 {
			logger.Error("parsing-command", errorspkg.New("id was not specified"))
			return newExitError("id was not specified", 1)
		}

		configBuilder := ctx.App.Metadata["configBuilder"].(*config.Builder)
		cfg, err := configBuilder.Build()
		logger.Debug("delete-config", lager.Data{"currentConfig": cfg})
		if err != nil {
			logger.Error("config-builder-failed", err)
			return newExitError(err.Error(), 1)
		}

		storePath := cfg.StorePath
		idOrPath := ctx.Args().First()
		id, err := idfinder.FindID(storePath, idOrPath)
		if err != nil {
			logger.Debug("id-not-found-skipping", lager.Data{"id": idOrPath, "storePath": storePath, "errorMessage": err.Error()})
			fmt.Println(err)
			return nil
		}

		fsDriver, err := createFileSystemDriver(cfg)
		if err != nil {
			logger.Error("failed-to-initialise-filesystem-driver", err)
			return newExitError(err.Error(), 1)
		}

		imageDriver, err := createImageDriver(cfg, fsDriver)
		if err != nil {
			logger.Error("failed-to-initialise-image-driver", err)
			return newExitError(err.Error(), 1)
		}

		imageCloner := image_cloner.NewImageCloner(imageDriver, storePath)
		dependencyManager := dependency_manager.NewDependencyManager(
			filepath.Join(storePath, store.MetaDirName, "dependencies"),
		)
		metricsEmitter := metrics.NewEmitter()
		deleter := groot.IamDeleter(imageCloner, dependencyManager, metricsEmitter)

		sm := store.NewStoreMeasurer(storePath, fsDriver)
		gc := garbage_collector.NewGC(fsDriver, imageCloner, dependencyManager)

		defer func() {
			unusedVols, err := gc.UnusedVolumes(logger, nil)
			if err != nil {
				logger.Error("getting-unused-layers-failed", err)
				return
			}
			metricsEmitter.TryEmitUsage(logger, "UnusedLayersSize", sm.CacheUsage(logger, unusedVols), "bytes")
		}()

		err = deleter.Delete(logger, id)
		if err != nil {
			logger.Error("deleting-image-failed", err)
			return newExitError(err.Error(), 1)
		}

		fmt.Printf("Image %s deleted\n", id)
		metricsEmitter.TryIncrementRunCount("delete", nil)
		return nil
	},
}

var DeleteStoreCommand = cli.Command{
	Name:        "delete-store",
	Usage:       "delete-store --store <path>",
	Description: "Deletes the given store from the system",

	Action: func(ctx *cli.Context) error {
		logger := ctx.App.Metadata["logger"].(lager.Logger)
		logger = logger.Session("delete-store")

		configBuilder := ctx.App.Metadata["configBuilder"].(*config.Builder)
		cfg, err := configBuilder.Build()
		logger.Debug("delete-store", lager.Data{"currentConfig": cfg})
		if err != nil {
			logger.Error("config-builder-failed", err)
			return cli.NewExitError(err.Error(), 1)
		}

		fsDriver, err := createFileSystemDriver(cfg)
		if err != nil {
			logger.Error("failed-to-initialise-filesystem-driver", err)
			return cli.NewExitError(err.Error(), 1)
		}

		storePath := cfg.StorePath
		locksmith := locksmith.NewSharedFileSystem(storePath, metrics.NewEmitter())
		manager := manager.New(storePath, nil, fsDriver, fsDriver, fsDriver)

		if err := manager.DeleteStore(logger, locksmith); err != nil {
			logger.Error("cleaning-up-store-failed", err)
			return cli.NewExitError(err.Error(), 1)
		}

		return nil
	},
}

var GenerateVolumeSizeMetadata = cli.Command{
	Name:   "generate-volume-size-metadata",
	Hidden: true,

	Action: func(ctx *cli.Context) error {
		logger := ctx.App.Metadata["logger"].(lager.Logger)
		logger = logger.Session("generate-metadata")

		if ctx.NArg() != 0 {
			logger.Error("parsing-command", errorspkg.New("invalid arguments"), lager.Data{"args": ctx.Args()})
			return cli.NewExitError(fmt.Sprintf("invalid arguments - usage: %s", ctx.Command.Usage), 1)
		}

		configBuilder := ctx.App.Metadata["configBuilder"].(*config.Builder)
		cfg, err := configBuilder.Build()
		if err != nil {
			return err
		}

		driver, err := createFileSystemDriver(cfg)
		if err != nil {
			return err
		}

		volumes, err := driver.Volumes(logger)
		if err != nil {
			return err
		}

		for _, volumeID := range volumes {
			_, err := driver.VolumeSize(logger, volumeID)
			if os.IsNotExist(errorspkg.Cause(err)) {
				logger.Info("volume-meta-missing", lager.Data{"volumeID": volumeID})

				volumePath, err := driver.VolumePath(logger, volumeID)
				if err != nil {
					return err
				}

				size, err := filesystems.CalculatePathSize(logger, volumePath)
				if err != nil {
					return err
				}

				err = driver.WriteVolumeMeta(logger, volumeID, base_image_puller.VolumeMeta{Size: size})
				if err != nil {
					return err
				}
			}
		}

		return nil
	},
}

var InitStoreCommand = cli.Command{
	Name:        "init-store",
	Usage:       "init-store --store <path>",
	Description: "Initialize a Store Directory",

	Flags: []cli.Flag{
		cli.StringSliceFlag{
			Name:  "uid-mapping",
			Usage: "UID mapping for image translation, e.g.: <Namespace UID>:<Host UID>:<Size>",
		},
		cli.StringSliceFlag{
			Name:  "gid-mapping",
			Usage: "GID mapping for image translation, e.g.: <Namespace GID>:<Host GID>:<Size>",
		},
		cli.Int64Flag{
			Name:  "store-size-bytes",
			Usage: "Creates a new filesystem of the given size and mounts it to the given Store Directory",
		},
	},

	Action: func(ctx *cli.Context) error {
		logger := ctx.App.Metadata["logger"].(lager.Logger)
		logger = logger.Session("init-store")

		if ctx.NArg() != 0 {
			logger.Error("parsing-command", errorspkg.New("invalid arguments"), lager.Data{"args": ctx.Args()})
			return cli.NewExitError(fmt.Sprintf("invalid arguments - usage: %s", ctx.Command.Usage), 1)
		}

		configBuilder := ctx.App.Metadata["configBuilder"].(*config.Builder).
			WithStoreSizeBytes(ctx.Int64("store-size-bytes"))
		cfg, err := configBuilder.Build()
		logger.Debug("init-store", lager.Data{"currentConfig": cfg})
		if err != nil {
			logger.Error("config-builder-failed", err)
			return cli.NewExitError(err.Error(), 1)
		}

		storePath := cfg.StorePath
		storeSizeBytes := cfg.Init.StoreSizeBytes

		if os.Getuid() != 0 {
			err := errorspkg.Errorf("store %s can only be initialized by Root user", storePath)
			logger.Error("init-store-failed", err)
			return cli.NewExitError(err.Error(), 1)
		}

		fsDriver, err := createFileSystemDriver(cfg)
		if err != nil {
			logger.Error("failed-to-initialise-filesystem-driver", err)
			return cli.NewExitError(err.Error(), 1)
		}

		uidMappings, err := parseIDMappings(ctx.StringSlice("uid-mapping"))
		if err != nil {
			err = errorspkg.Errorf("parsing uid-mapping: %s", err)
			logger.Error("parsing-command", err)
			return cli.NewExitError(err.Error(), 1)
		}
		gidMappings, err := parseIDMappings(ctx.StringSlice("gid-mapping"))
		if err != nil {
			err = errorspkg.Errorf("parsing gid-mapping: %s", err)
			logger.Error("parsing-command", err)
			return cli.NewExitError(err.Error(), 1)
		}

		namespacer := groot.NewStoreNamespacer(storePath)
		spec := manager.InitSpec{
			UIDMappings:    uidMappings,
			GIDMappings:    gidMappings,
			StoreSizeBytes: storeSizeBytes,
		}

		manager := manager.New(storePath, namespacer, fsDriver, fsDriver, fsDriver)
		if err := manager.InitStore(logger, spec); err != nil {
			logger.Error("cleaning-up-store-failed", err)
			return cli.NewExitError(errorspkg.Cause(err).Error(), 1)
		}

		return nil
	},
}

var ListCommand = cli.Command{
	Name:        "list",
	Usage:       "list",
	Description: "Lists images in store",

	Action: func(ctx *cli.Context) error {
		logger := ctx.App.Metadata["logger"].(lager.Logger)
		logger = logger.Session("list")

		configBuilder := ctx.App.Metadata["configBuilder"].(*config.Builder)
		cfg, err := configBuilder.Build()
		logger.Debug("list-config", lager.Data{"currentConfig": cfg})
		if err != nil {
			logger.Error("config-builder-failed", err)
			return cli.NewExitError(err.Error(), 1)
		}

		if _, err := os.Stat(cfg.StorePath); os.IsNotExist(err) {
			err := errorspkg.Errorf("no store found at %s", cfg.StorePath)
			logger.Error("store-path-failed", err, nil)
			return cli.NewExitError(err.Error(), 1)
		}

		lister := groot.IamLister()
		images, err := lister.List(logger, cfg.StorePath)
		if err != nil {
			logger.Error("listing-images", err, lager.Data{"storePath": cfg.StorePath})
			return cli.NewExitError(fmt.Sprintf("Failed to retrieve list of images: %s", err.Error()), 1)
		}

		if len(images) == 0 {
			fmt.Println("Store empty")
		}
		for _, image := range images {
			fmt.Println(image)
		}

		return nil
	},
}

var StatsCommand = cli.Command{
	Name:        "stats",
	Usage:       "stats [options] <id|image path>",
	Description: "Return filesystem stats",

	Action: func(ctx *cli.Context) error {
		logger := ctx.App.Metadata["logger"].(lager.Logger)
		logger = logger.Session("stats")
		newExitError := newErrorHandler(logger, "stats")

		if ctx.NArg() != 1 {
			logger.Error("parsing-command", errorspkg.New("invalid arguments"), lager.Data{"args": ctx.Args()})
			return newExitError(fmt.Sprintf("invalid arguments - usage: %s", ctx.Command.Usage), 1)
		}

		configBuilder := ctx.App.Metadata["configBuilder"].(*config.Builder)
		cfg, err := configBuilder.Build()
		logger.Debug("stats-config", lager.Data{"currentConfig": cfg})
		if err != nil {
			logger.Error("config-builder-failed", err)
			return newExitError(err.Error(), 1)
		}

		storePath := cfg.StorePath
		idOrPath := ctx.Args().First()
		id, err := idfinder.FindID(storePath, idOrPath)
		if err != nil {
			logger.Error("find-id-failed", err, lager.Data{"id": idOrPath, "storePath": storePath})
			return newExitError(err.Error(), 1)
		}

		fsDriver, err := createFileSystemDriver(cfg)
		if err != nil {
			return newExitError(err.Error(), 1)
		}
		imageCloner := imageClonerpkg.NewImageCloner(fsDriver, storePath)

		metricsEmitter := metrics.NewEmitter()
		statser := groot.IamStatser(imageCloner, metricsEmitter)
		stats, err := statser.Stats(logger, id)
		if err != nil {
			logger.Error("fetching-stats", err)
			return newExitError(err.Error(), 1)
		}

		_ = json.NewEncoder(os.Stdout).Encode(stats)
		metricsEmitter.TryIncrementRunCount("stats", nil)
		return nil
	},
}