cmdgc

Documentation

Index

• Variables

Variables

var Command = &command.C{
	Name: "gc",
	Help: `Garbage-collect objects not reachable from known roots.

If no roots are defined, an error is reported without making any changes
unless -force is set. This avoids accidentally deleting everything in a
store without roots.
`,

	SetFlags: func(_ *command.Env, fs *flag.FlagSet) { flax.MustBind(fs, &gcFlags) },

	Run: func(env *command.Env) error {
		if len(env.Args) != 0 {
			return env.Usagef("extra arguments after command")
		}

		cfg := env.Config.(*config.Settings)
		return cfg.WithStore(cfg.Context, func(s config.CAS) error {
			var keys []string
			if err := s.Roots().List(cfg.Context, "", func(key string) error {
				keys = append(keys, key)
				return nil
			}); err != nil {
				return fmt.Errorf("listing roots: %w", err)
			}

			if len(keys) == 0 && !gcFlags.Force {
				return errors.New("there are no root keys defined")
			} else if len(keys) == 0 {
				fmt.Fprint(env, `>> WARNING <<
* No root keys found!
* Proceeding with collection anyway because -force is set

`)
			}

			n, err := s.Len(cfg.Context)
			if err != nil {
				return err
			} else if n == 0 {
				return errors.New("the store is empty")
			}
			var idxs []*index.Index
			idx := index.New(int(n), &index.Options{FalsePositiveRate: 0.01})
			fmt.Fprintf(env, "Begin GC of %d objects, roots=%+q\n", n, keys)

			for i := 0; i < len(keys); i++ {
				key := keys[i]
				rp, err := root.Open(cfg.Context, s.Roots(), key)
				if err != nil {
					return fmt.Errorf("opening %q: %w", key, err)
				}
				idx.Add(key)

				if rp.IndexKey != "" {
					rpi, err := config.LoadIndex(cfg.Context, s, rp.IndexKey)
					if err != nil {
						return err
					}
					idxs = append(idxs, rpi)
					idx.Add(rp.IndexKey)
					fmt.Fprintf(env, "Loaded cached index for %q (%x)\n", key, rp.IndexKey)
					continue
				}

				rf, err := rp.File(cfg.Context, s)
				if err != nil {
					return fmt.Errorf("opening %q: %w", rp.FileKey, err)
				}
				idx.Add(rp.FileKey)

				fmt.Fprintf(env, "Scanning data reachable from %q (%x)...\n",
					config.PrintableKey(key), rp.FileKey)
				scanned := mapset.New[string]()
				start := time.Now()
				if err := rf.Scan(cfg.Context, func(si file.ScanItem) bool {
					key := si.Key()
					if scanned.Has(key) {
						return false
					}
					scanned.Add(key)
					idx.Add(key)
					for _, dkey := range si.Data().Keys() {
						idx.Add(dkey)
					}
					return true
				}); err != nil {
					return fmt.Errorf("scanning %q: %w", key, err)
				}
				fmt.Fprintf(env, "Finished scanning %d objects [%v elapsed]\n",
					idx.Len(), time.Since(start).Truncate(10*time.Millisecond))
			}
			idxs = append(idxs, idx)

			ctx, cancel := context.WithCancelCause(cfg.Context)
			defer cancel(nil)
			if gcFlags.Limit > 0 {
				fmt.Fprintf(env, "- sweep limit %v\n", gcFlags.Limit)
				time.AfterFunc(gcFlags.Limit, func() { cancel(errSweepLimit) })
			}
			g, run := taskgroup.New(taskgroup.Listen(cancel)).Limit(64)

			fmt.Fprintf(env, "Begin sweep over %d objects...\n", n)
			start := time.Now()
			var numKeep, numDrop atomic.Int64
			for i := 0; i < 256; i++ {
				pfx := string([]byte{byte(i)})
				g.Go(func() error {
					return s.List(ctx, pfx, func(key string) error {
						if !strings.HasPrefix(key, pfx) {
							return blob.ErrStopListing
						}
						run(taskgroup.NoError(func() {
							for _, idx := range idxs {
								if idx.Has(key) {
									numKeep.Add(1)
									return
								}
							}
							if numDrop.Add(1)%50 == 0 {
								fmt.Fprint(env, ".")
							}
							if err := s.Delete(ctx, key); err != nil && !errors.Is(err, context.Canceled) {
								log.Printf("WARNING: delete key %x: %v", key, err)
							}
						}))
						return nil
					})
				})
			}
			serr := g.Wait()
			fmt.Fprintln(env, "*")
			if serr != nil {
				if errors.Is(context.Cause(ctx), errSweepLimit) {
					fmt.Fprintln(env, "(sweep limit reached)")
				} else {
					return fmt.Errorf("sweeping failed: %w", serr)
				}
			}
			fmt.Fprintf(env, "GC complete: keep %d, drop %d [%v elapsed]\n",
				numKeep.Load(), numDrop.Load(), time.Since(start).Truncate(10*time.Millisecond))
			return nil
		})
	},
}