cmdgc

Documentation

Index

• Variables

Variables

var Command = &command.C{
	Name: "gc",
	Help: `Garbage-collect objects not reachable from known roots.

If no roots are defined, an error is reported without making any changes
unless -force is set. This avoids accidentally deleting everything in a
store without roots.
`,

	SetFlags: func(_ *command.Env, fs *flag.FlagSet) { flax.MustBind(fs, &gcFlags) },

	Run: func(env *command.Env, args []string) error {
		if len(args) != 0 {
			return env.Usagef("extra arguments after command")
		} else if gcFlags.Partial <= 0 || gcFlags.Partial > 1 {
			return errors.New("sweep fraction must be in 0..1")
		}

		cfg := env.Config.(*config.Settings)
		ctx, cancel := context.WithCancel(cfg.Context)
		return cfg.WithStore(cfg.Context, func(s config.CAS) error {
			var keys []string
			if err := s.Roots().List(cfg.Context, "", func(key string) error {
				keys = append(keys, key)
				return nil
			}); err != nil {
				return fmt.Errorf("listing roots: %w", err)
			}

			if len(keys) == 0 && !gcFlags.Force {
				return errors.New("there are no root keys defined")
			} else if len(keys) == 0 {
				fmt.Fprint(env, `>> WARNING <<
* No root keys found!
* Proceeding with collection anyway because -force is set

`)
			}

			n, err := s.Len(ctx)
			if err != nil {
				return err
			} else if n == 0 {
				return errors.New("the store is empty")
			}
			var idxs []*index.Index
			idx := index.New(int(n), &index.Options{FalsePositiveRate: 0.01})
			fmt.Fprintf(env, "Begin GC of %d objects, roots=%+q\n", n, keys)

			for i := 0; i < len(keys); i++ {
				key := keys[i]
				rp, err := root.Open(cfg.Context, s.Roots(), key)
				if err != nil {
					return fmt.Errorf("opening %q: %w", key, err)
				}
				idx.Add(key)

				if rp.IndexKey != "" {
					rpi, err := config.LoadIndex(cfg.Context, s, rp.IndexKey)
					if err != nil {
						return err
					}
					idxs = append(idxs, rpi)
					idx.Add(rp.IndexKey)
					fmt.Fprintf(env, "Loaded cached index for %q (%x)\n", key, rp.IndexKey)
					continue
				}

				rf, err := rp.File(cfg.Context, s)
				if err != nil {
					return fmt.Errorf("opening %q: %w", rp.FileKey, err)
				}
				idx.Add(rp.FileKey)

				fmt.Fprintf(env, "Scanning data reachable from %q (%x)...\n",
					config.PrintableKey(key), rp.FileKey)
				scanned := mapset.New[string]()
				start := time.Now()
				if err := rf.Scan(cfg.Context, func(si file.ScanItem) bool {
					key := si.Key()
					if scanned.Has(key) {
						return false
					}
					scanned.Add(key)
					idx.Add(key)
					for _, dkey := range si.Data().Keys() {
						idx.Add(dkey)
					}
					return true
				}); err != nil {
					return fmt.Errorf("scanning %q: %w", key, err)
				}
				fmt.Fprintf(env, "Finished scanning %d objects [%v elapsed]\n",
					idx.Len(), time.Since(start).Truncate(10*time.Millisecond))
			}
			idxs = append(idxs, idx)

			g, run := taskgroup.New(taskgroup.Trigger(cancel)).Limit(64)

			fmt.Fprintf(env, "Begin sweep over %d objects...\n", n)
			sample := func() bool { return true }
			if gcFlags.Partial < 1 {
				rng := rand.New(rand.NewSource(20230405090527))
				sample = func() bool { return rng.Float64() <= gcFlags.Partial }
				fmt.Fprintf(env, "- partial sweep with fraction %.2g\n", gcFlags.Partial)
			}
			start := time.Now()
			var numKeep, numDrop atomic.Int64
			for i := 0; i < 256; i++ {
				pfx := string([]byte{byte(i)})
				g.Go(func() error {
					return s.List(cfg.Context, pfx, func(key string) error {
						if !strings.HasPrefix(key, pfx) {
							return blob.ErrStopListing
						}
						run(func() error {
							for _, idx := range idxs {
								if idx.Has(key) {
									numKeep.Add(1)
									return nil
								}
							}
							if !sample() {
								return nil
							} else if numDrop.Add(1)%50 == 0 {
								fmt.Fprint(env, ".")
							}
							if err := s.Delete(ctx, key); err != nil && !errors.Is(err, context.Canceled) {
								log.Printf("WARNING: delete key %x: %v", key, err)
							}
							return nil
						})
						return nil
					})
				})
			}
			if err := g.Wait(); err != nil {
				return fmt.Errorf("sweeping failed: %w", err)
			}
			fmt.Fprintln(env, "*")
			fmt.Fprintf(env, "GC complete: keep %d, drop %d [%v elapsed]\n",
				numKeep.Load(), numDrop.Load(), time.Since(start).Truncate(10*time.Millisecond))
			return nil
		})
	},
}