josie / simplegit

gc: audit data dir, clean orphan uploads, opt-in vacuum

simplegit gc reports db/wal sizes, repo rows without dirs and repo
dirs without rows, orphan upload dirs/files from crashed publishes,
and expired sessions. --clean deletes only orphan uploads and expired
sessions (bare repo dirs with no row are reported, never deleted:
file:// pushes create them by design). --vacuum rewrites the DB and
checkpoints the WAL. db gains ListAllRepos + CountExpiredSessions.
docs: maintenance section. 3 new tests (148 total, vet+gofmt clean).

LLM Contributor: qwen/qwen3.8-flash

e17a0b8f951af062eac19c12c5c59fe429d959b5
cjosie <administrator@josie-c.com> · 2026-10-06T20:39 · browse files at this commit

parents: 569340c

diff --git a/cmd/simplegit/gc.go b/cmd/simplegit/gc.go
new file mode 100644
index 0000000..6c79bdc
--- /dev/null
+++ b/cmd/simplegit/gc.go
@@ -0,0 +1,261 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+
+	"git.josie-c.com/josie/simplegit/internal/db"
+)
+
+type gcItem struct {
+	path  string
+	size  int64
+	files int
+}
+
+// runGC audits the data dir: sizes, DB-vs-disk consistency, and orphaned
+// upload files — the one thing a crash between row-delete and file-delete
+// can strand. Read-only by default; --clean deletes orphan uploads and
+// expired sessions. Bare repo dirs without rows are reported, never deleted:
+// file:// pushes legitimately create them. --vacuum rewrites the DB.
+func runGC(args []string, w io.Writer) error {
+	fs := flag.NewFlagSet("gc", flag.ExitOnError)
+	configPath := fs.String("config", "simplegit.toml", "path to TOML config file")
+	clean := fs.Bool("clean", false, "delete orphan upload dirs/files and expired sessions")
+	vacuum := fs.Bool("vacuum", false, "rewrite the DB file and checkpoint the WAL (run while serve is stopped)")
+	fs.Usage = func() {
+		fmt.Fprintf(fs.Output(), "usage: simplegit gc [-config FILE] [--clean] [--vacuum]\n")
+		fs.PrintDefaults()
+	}
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+	if fs.NArg() != 0 {
+		return errors.New("gc takes no positional arguments")
+	}
+
+	cfg, database, err := openStore(*configPath)
+	if err != nil {
+		return err
+	}
+	defer database.Close()
+	ctx := context.Background()
+
+	repos, err := db.ListAllRepos(database)
+	if err != nil {
+		return err
+	}
+
+	fmt.Fprintf(w, "data dir: %s\n", cfg.DataDir)
+
+	dbPath := filepath.Join(cfg.DataDir, "simplegit.db")
+	walPath := dbPath + "-wal"
+	dbSize, walSize := fileSize(dbPath), fileSize(walPath)
+	if dbSize < 0 {
+		return fmt.Errorf("database %s not found (wrong config?)", dbPath)
+	}
+	fmt.Fprintf(w, "db: %s (+wal %s)\n", fmtBytes(dbSize), fmtBytes(walSize))
+
+	fmt.Fprintf(w, "repos: %d row(s)\n", len(repos))
+	repoPaths := make(map[string]bool, len(repos))
+	for _, repo := range repos {
+		key := filepath.Join(repo.OwnerName, repo.Name)
+		repoPaths[key] = true
+		if fileSize(filepath.Join(cfg.DataDir, "repos", key+".git")) < 0 {
+			fmt.Fprintf(w, "  ! repo row %s/%s has no directory on disk\n", repo.OwnerName, repo.Name)
+		}
+	}
+	repoRoot := filepath.Join(cfg.DataDir, "repos")
+	repoDisk, repoDiskSize := dirSize(repoRoot)
+	orphanRepos := 0
+	if entries, err := os.ReadDir(repoRoot); err == nil {
+		for _, owner := range entries {
+			ownerDir := filepath.Join(repoRoot, owner.Name())
+			gitDirs, err := os.ReadDir(ownerDir)
+			if err != nil || !owner.IsDir() {
+				continue
+			}
+			for _, entry := range gitDirs {
+				name := entry.Name()
+				if !entry.IsDir() || filepath.Ext(name) != ".git" {
+					continue
+				}
+				if !repoPaths[filepath.Join(owner.Name(), strings.TrimSuffix(name, ".git"))] {
+					fmt.Fprintf(w, "  ! repo dir %s/%s has no DB row (kept — out-of-band push?)\n", owner.Name(), name)
+					orphanRepos++
+				}
+			}
+		}
+	} else if !errors.Is(err, os.ErrNotExist) {
+		return err
+	}
+	fmt.Fprintf(w, "       %d dir(s) on disk, %s\n", repoDisk, fmtBytes(repoDiskSize))
+
+	liveReleases := map[int64]map[string]bool{}
+	var orphanDirs, orphanFiles []gcItem
+	var assetCount, releaseCount int
+	var uploadSize int64
+	releaseRoot := filepath.Join(cfg.DataDir, "uploads", "releases")
+	for _, repo := range repos {
+		releases, err := db.ListReleases(database, repo.ID)
+		if err != nil {
+			return err
+		}
+		for _, release := range releases {
+			releaseCount++
+			assets, err := db.ListReleaseAssets(database, release.ID)
+			if err != nil {
+				return err
+			}
+			stored := make(map[string]bool, len(assets))
+			for _, asset := range assets {
+				assetCount++
+				stored[asset.StoredName] = true
+				p := filepath.Join(releaseRoot, strconv.FormatInt(release.ID, 10), asset.StoredName)
+				if size := fileSize(p); size >= 0 {
+					uploadSize += size
+				} else {
+					fmt.Fprintf(w, "  ! asset %d (%s) of %s/%s@%s has no file on disk\n",
+						asset.ID, asset.Filename, repo.OwnerName, repo.Name, release.Tag)
+				}
+			}
+			liveReleases[release.ID] = stored
+		}
+	}
+	if entries, err := os.ReadDir(releaseRoot); err == nil {
+		for _, entry := range entries {
+			id, err := strconv.ParseInt(entry.Name(), 10, 64)
+			full := filepath.Join(releaseRoot, entry.Name())
+			if !entry.IsDir() || err != nil {
+				orphanDirs = append(orphanDirs, dirItem(full))
+				continue
+			}
+			if _, live := liveReleases[id]; !live {
+				orphanDirs = append(orphanDirs, dirItem(full))
+				continue
+			}
+			files, err := os.ReadDir(full)
+			if err != nil {
+				return err
+			}
+			for _, f := range files {
+				if !f.IsDir() && !liveReleases[id][f.Name()] {
+					orphanFiles = append(orphanFiles, gcItem{path: filepath.Join(full, f.Name()), size: fileSizeOf(f)})
+				}
+			}
+		}
+	} else if !errors.Is(err, os.ErrNotExist) {
+		return err
+	}
+	fmt.Fprintf(w, "uploads: %d release(s), %d asset row(s)\n", releaseCount, assetCount)
+	allOrphans := append(append([]gcItem{}, orphanDirs...), orphanFiles...)
+	for _, item := range allOrphans {
+		fmt.Fprintf(w, "  ! orphan %s (%d file(s), %s)\n", item.path, item.files, fmtBytes(item.size))
+	}
+	for _, item := range allOrphans {
+		uploadSize += item.size
+	}
+	fmt.Fprintf(w, "         %s total\n", fmtBytes(uploadSize))
+
+	expired, err := db.CountExpiredSessions(database)
+	if err != nil {
+		return err
+	}
+	fmt.Fprintf(w, "sessions: %d expired\n", expired)
+
+	orphanTotal := len(allOrphans)
+	switch {
+	case *clean:
+		for _, item := range allOrphans {
+			if err := os.RemoveAll(item.path); err != nil {
+				return fmt.Errorf("remove %s: %w", item.path, err)
+			}
+			fmt.Fprintf(w, "removed %s\n", item.path)
+		}
+		if expired > 0 {
+			if err := db.DeleteExpiredSessions(database); err != nil {
+				return err
+			}
+			fmt.Fprintf(w, "purged %d expired session(s)\n", expired)
+		}
+	case orphanTotal > 0 || expired > 0:
+		fmt.Fprintln(w, "dry run: pass --clean to delete orphan uploads and expired sessions")
+	default:
+		fmt.Fprintln(w, "no orphan uploads or expired sessions")
+	}
+
+	if orphanRepos > 0 {
+		fmt.Fprintln(w, "note: orphan repo dirs are never deleted by gc (file:// pushes create them by design)")
+	}
+
+	if *vacuum {
+		before := fileSize(dbPath)
+		if _, err := database.ExecContext(ctx, `VACUUM`); err != nil {
+			return fmt.Errorf("vacuum: %w", err)
+		}
+		if _, err := database.ExecContext(ctx, `PRAGMA wal_checkpoint(TRUNCATE)`); err != nil {
+			return fmt.Errorf("wal checkpoint: %w", err)
+		}
+		fmt.Fprintf(w, "vacuum: db %s -> %s\n", fmtBytes(before), fmtBytes(fileSize(dbPath)))
+	}
+	return nil
+}
+
+// fileSize returns -1 when the path is absent or unreadable.
+func fileSize(path string) int64 {
+	info, err := os.Stat(path)
+	if err != nil {
+		return -1
+	}
+	return info.Size()
+}
+
+func fileSizeOf(dirEntry os.DirEntry) int64 {
+	info, err := dirEntry.Info()
+	if err != nil {
+		return 0
+	}
+	return info.Size()
+}
+
+func dirItem(path string) gcItem {
+	items, size := dirSize(path)
+	return gcItem{path: path, size: size, files: items}
+}
+
+func dirSize(root string) (int, int64) {
+	var files int
+	var size int64
+	filepath.WalkDir(root, func(_ string, d os.DirEntry, err error) error {
+		if err == nil && !d.IsDir() {
+			files++
+			size += fileSizeOf(d)
+		}
+		return nil
+	})
+	return files, size
+}
+
+func fmtBytes(n int64) string {
+	if n < 0 {
+		return "?"
+	}
+	if n < 1024 {
+		return fmt.Sprintf("%d B", n)
+	}
+	v := float64(n)
+	for _, unit := range []string{"KB", "MB", "GB", "TB"} {
+		v /= 1024
+		if v < 1024 || unit == "TB" {
+			return fmt.Sprintf("%.1f %s", v, unit)
+		}
+	}
+	return "?"
+}
diff --git a/cmd/simplegit/gc_test.go b/cmd/simplegit/gc_test.go
new file mode 100644
index 0000000..19f7038
--- /dev/null
+++ b/cmd/simplegit/gc_test.go
@@ -0,0 +1,188 @@
+package main
+
+import (
+	"bytes"
+	"database/sql"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	"git.josie-c.com/josie/simplegit/internal/db"
+)
+
+// gcFixture builds a data dir with one healthy repo (row+dir), a ghost repo
+// row, an out-of-band repo dir, a live release with a real asset plus a
+// stranded file, an orphan release dir, a stray file at the releases root,
+// and one expired session.
+func gcFixture(t *testing.T) (dataDir, configPath string, database *sql.DB) {
+	t.Helper()
+	dataDir = t.TempDir()
+	repoPath := filepath.Join(dataDir, "repos", "josie", "demo.git")
+	if err := os.MkdirAll(repoPath, 0o755); err != nil {
+		t.Fatalf("mkdir: %v", err)
+	}
+	database = openDB(t, dataDir)
+
+	ownerID, err := db.CreateUser(database, "josie", "hash")
+	if err != nil {
+		t.Fatalf("CreateUser: %v", err)
+	}
+	demo, err := db.CreateRepo(database, ownerID, "demo", "", "public")
+	if err != nil {
+		t.Fatalf("CreateRepo: %v", err)
+	}
+	if _, err := db.CreateRepo(database, ownerID, "ghost", "", "public"); err != nil {
+		t.Fatalf("CreateRepo ghost: %v", err)
+	}
+	if err := os.MkdirAll(filepath.Join(dataDir, "repos", "josie", "leftover.git"), 0o755); err != nil {
+		t.Fatalf("mkdir leftover: %v", err)
+	}
+
+	release, err := db.CreateRelease(database, demo.ID, "v1", "V1", "", ownerID)
+	if err != nil {
+		t.Fatalf("CreateRelease: %v", err)
+	}
+	if _, err := db.CreateReleaseAsset(database, release.ID, "f.bin", "opaque1", 4); err != nil {
+		t.Fatalf("CreateReleaseAsset: %v", err)
+	}
+	releaseRoot := filepath.Join(dataDir, "uploads", "releases")
+	liveDir := filepath.Join(releaseRoot, strconv.FormatInt(release.ID, 10))
+	writeFile(t, filepath.Join(liveDir, "opaque1"), "data")
+	writeFile(t, filepath.Join(liveDir, "stray123"), "stranded")
+	writeFile(t, filepath.Join(releaseRoot, "999", "orphan.bin"), "orphan")
+	writeFile(t, filepath.Join(releaseRoot, "junk.txt"), "stray file")
+
+	if err := db.CreateSession(database, "deadtoken", ownerID, time.Now().Unix()-60); err != nil {
+		t.Fatalf("CreateSession: %v", err)
+	}
+
+	configPath = filepath.Join(t.TempDir(), "gc.toml")
+	writeFile(t, configPath, "data_dir = "+strconv.Quote(dataDir)+"\n")
+	return dataDir, configPath, database
+}
+
+func openDB(t *testing.T, dataDir string) *sql.DB {
+	t.Helper()
+	database, err := db.Open(filepath.Join(dataDir, "simplegit.db"))
+	if err != nil {
+		t.Fatalf("db.Open: %v", err)
+	}
+	t.Cleanup(func() { database.Close() })
+	return database
+}
+
+func writeFile(t *testing.T, path, content string) {
+	t.Helper()
+	if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
+		t.Fatalf("mkdir %s: %v", filepath.Dir(path), err)
+	}
+	if err := os.WriteFile(path, []byte(content), 0o600); err != nil {
+		t.Fatalf("write %s: %v", path, err)
+	}
+}
+
+func runGCText(t *testing.T, configPath string, extra ...string) string {
+	t.Helper()
+	var out bytes.Buffer
+	args := append([]string{"-config", configPath}, extra...)
+	if err := runGC(args, &out); err != nil {
+		t.Fatalf("runGC(%v): %v", extra, err)
+	}
+	return out.String()
+}
+
+func TestGCDryRunReportsAndKeeps(t *testing.T) {
+	dataDir, configPath, _ := gcFixture(t)
+
+	out := runGCText(t, configPath)
+	for _, want := range []string{
+		"repo row josie/ghost has no directory",
+		"repo dir josie/leftover.git has no DB row",
+		"orphan", "stray123", "999", "junk.txt",
+		"sessions: 1 expired",
+		"dry run: pass --clean",
+	} {
+		if !strings.Contains(out, want) {
+			t.Errorf("report missing %q:\n%s", want, out)
+		}
+	}
+
+	releaseRoot := filepath.Join(dataDir, "uploads", "releases")
+	for _, kept := range []string{"999", "junk.txt"} {
+		if _, err := os.Stat(filepath.Join(releaseRoot, kept)); err != nil {
+			t.Errorf("dry run deleted %s: %v", kept, err)
+		}
+	}
+	liveDir, err := os.ReadDir(releaseRoot)
+	if err != nil {
+		t.Fatalf("readdir: %v", err)
+	}
+	for _, entry := range liveDir {
+		if entry.Name() == "999" || !entry.IsDir() {
+			continue
+		}
+		if files, _ := os.ReadDir(filepath.Join(releaseRoot, entry.Name())); len(files) != 2 {
+			t.Errorf("dry run touched live release dir %s: %v", entry.Name(), files)
+		}
+	}
+}
+
+func TestGCCleanRemovesOnlyOrphanUploadsAndSessions(t *testing.T) {
+	dataDir, configPath, database := gcFixture(t)
+
+	out := runGCText(t, configPath, "--clean")
+	if !strings.Contains(out, "removed") || !strings.Contains(out, "purged 1 expired session") {
+		t.Fatalf("unexpected clean output:\n%s", out)
+	}
+
+	releaseRoot := filepath.Join(dataDir, "uploads", "releases")
+	if _, err := os.Stat(filepath.Join(releaseRoot, "999")); !os.IsNotExist(err) {
+		t.Error("orphan release dir survived --clean")
+	}
+	if _, err := os.Stat(filepath.Join(releaseRoot, "junk.txt")); !os.IsNotExist(err) {
+		t.Error("stray file survived --clean")
+	}
+	liveDir := filepath.Join(releaseRoot, "1")
+	if _, err := os.Stat(filepath.Join(liveDir, "opaque1")); err != nil {
+		t.Errorf("live asset deleted by --clean: %v", err)
+	}
+	if _, err := os.Stat(filepath.Join(liveDir, "stray123")); !os.IsNotExist(err) {
+		t.Error("stranded file in live dir survived --clean")
+	}
+	if _, err := os.Stat(filepath.Join(dataDir, "repos", "josie", "leftover.git")); err != nil {
+		t.Errorf("--clean must never delete repo dirs: %v", err)
+	}
+
+	if n, err := db.CountExpiredSessions(database); err != nil || n != 0 {
+		t.Errorf("expired sessions after clean: n=%d err=%v", n, err)
+	}
+}
+
+func TestGCVacuum(t *testing.T) {
+	dataDir := t.TempDir()
+	database := openDB(t, dataDir)
+	if _, err := db.CreateUser(database, "josie", "hash"); err != nil {
+		t.Fatalf("CreateUser: %v", err)
+	}
+	for i := 0; i < 200; i++ {
+		if err := db.CreateSession(database, strconv.Itoa(i), 1, time.Now().Unix()+60); err != nil {
+			t.Fatalf("CreateSession: %v", err)
+		}
+	}
+	if err := db.DeleteExpiredSessions(database); err != nil {
+		t.Fatalf("DeleteExpiredSessions: %v", err)
+	}
+	configPath := filepath.Join(t.TempDir(), "gc.toml")
+	writeFile(t, configPath, "data_dir = "+strconv.Quote(dataDir)+"\n")
+
+	out := runGCText(t, configPath, "--vacuum")
+	if !strings.Contains(out, "vacuum: db") {
+		t.Fatalf("no vacuum line:\n%s", out)
+	}
+	if _, err := db.GetUserByName(database, "josie"); err != nil {
+		t.Errorf("db unreadable after vacuum: %v", err)
+	}
+}
diff --git a/cmd/simplegit/main.go b/cmd/simplegit/main.go
index 9aa2456..bfc0bef 100644
--- a/cmd/simplegit/main.go
+++ b/cmd/simplegit/main.go
@@ -44,6 +44,8 @@ func main() {
 		err = runRelease(os.Args[2:])
 	case "hook":
 		err = runHook(os.Args[2:])
+	case "gc":
+		err = runGC(os.Args[2:], os.Stdout)
 	case "-h", "--help", "help":
 		usage()
 		return
@@ -66,6 +68,7 @@ commands:
   serve     run the HTTP server
   adduser   create the account
   release   publish a release (owner, via the server's API)
+  gc        audit data-dir sizes and orphaned files (--clean, --vacuum)
   hook      post-receive callback (invoked by git, not by hand)
 `)
 }
diff --git a/docs/self-host.md b/docs/self-host.md
index ea9cc71..e5cf8d4 100644
--- a/docs/self-host.md
+++ b/docs/self-host.md
@@ -272,3 +272,26 @@ Everything lives under `data_dir`:
 Back up all three together; the DB and uploads reference each other. SQLite
 is in WAL mode, so snapshot with `sqlite3 simplegit.db ".backup <dest>"` (or
 copy the DB plus its `-wal`) rather than copying a live file blindly.
+
+## 9. Maintenance (gc)
+
+`simplegit gc` audits `data_dir`: DB size, repo counts, release upload sizes,
+DB↔disk inconsistencies (repo rows without dirs, asset rows without files),
+orphaned upload files/dirs from crashed publishes, and expired sessions.
+
+```sh
+simplegit gc                 # read-only report
+simplegit gc --clean         # also delete orphan uploads + expired sessions
+simplegit gc --vacuum        # rewrite the DB, checkpoint the WAL
+```
+
+`--vacuum` is best run while `serve` is stopped:
+
+```sh
+systemctl stop simplegit
+simplegit gc --clean --vacuum
+systemctl start simplegit
+```
+
+gc never deletes bare repo directories that lack a DB row (an out-of-band
+`git init` by hand is not garbage it should auto-remove) — it reports them.
diff --git a/internal/db/repos.go b/internal/db/repos.go
index d1853b8..e367462 100644
--- a/internal/db/repos.go
+++ b/internal/db/repos.go
@@ -118,6 +118,20 @@ func ListRepos(database *sql.DB, viewerID int64) ([]RepoView, error) {
 	return repos, nil
 }
 
+// ListAllRepos returns every repo regardless of visibility; ListRepos is
+// viewer-scoped, so it cannot give a full inventory.
+func ListAllRepos(database *sql.DB) ([]RepoView, error) {
+	repos, err := listQuery(database,
+		`SELECT r.id, r.owner_id, r.name, r.description, r.visibility,
+		        r.default_branch, r.pushed_at, u.username
+		 FROM repos r JOIN users u ON u.id = r.owner_id
+		 ORDER BY u.username, r.name`, nil, scanRepoView)
+	if err != nil {
+		return nil, fmt.Errorf("list all repos: %w", err)
+	}
+	return repos, nil
+}
+
 // scanRepoView reads one repos+owner row (see ListRepos) into view.
 func scanRepoView(s rowScanner, view *RepoView) error {
 	return s.Scan(&view.ID, &view.OwnerID, &view.Name, &view.Description,
diff --git a/internal/db/sessions.go b/internal/db/sessions.go
index d5f9c9e..514bc59 100644
--- a/internal/db/sessions.go
+++ b/internal/db/sessions.go
@@ -63,6 +63,15 @@ func DeleteOtherSessions(database *sql.DB, userID int64, keepToken string) error
 	return nil
 }
 
+// CountExpiredSessions returns the number of sessions past their expiry.
+func CountExpiredSessions(database *sql.DB) (int, error) {
+	var n int
+	if err := database.QueryRow(`SELECT count(*) FROM sessions WHERE expires_at <= unixepoch()`).Scan(&n); err != nil {
+		return 0, fmt.Errorf("count expired sessions: %w", err)
+	}
+	return n, nil
+}
+
 // DeleteExpiredSessions purges sessions whose expiry has passed; callers
 // use it as opportunistic housekeeping, so no error on zero rows.
 func DeleteExpiredSessions(database *sql.DB) error {