From aac44e9629eaa93338f87d666fec32e9407ea6ed Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Thu, 23 Jul 2026 08:43:30 +0200 Subject: [PATCH 1/2] fix: speed up updates and show progress --- .changeset/faster-visible-updates.md | 5 + Dockerfile | 10 +- go/cmd/ftw-updater/main.go | 107 +++++++++++++++--- go/cmd/ftw-updater/main_test.go | 26 ++++- go/internal/api/api_selfupdate.go | 131 +++++++++++++++++++--- go/internal/api/api_selfupdate_test.go | 27 +++++ go/internal/api/snapshots.go | 11 +- go/internal/selfupdate/selfupdate.go | 33 +++++- go/internal/selfupdate/selfupdate_test.go | 56 ++++++++- go/internal/state/store.go | 73 +++++++++++- go/internal/state/store_test.go | 31 +++++ scripts/test-container-boundaries.sh | 5 + web/components/ftw-update-check.js | 86 +++++++++++--- web/update-badge.js | 117 ++++++++++++++++++- web/update-progress.test.mjs | 32 ++++++ 15 files changed, 681 insertions(+), 69 deletions(-) create mode 100644 .changeset/faster-visible-updates.md create mode 100644 web/update-progress.test.mjs diff --git a/.changeset/faster-visible-updates.md b/.changeset/faster-visible-updates.md new file mode 100644 index 00000000..610e0c02 --- /dev/null +++ b/.changeset/faster-visible-updates.md @@ -0,0 +1,5 @@ +--- +"ftw": patch +--- + +Reduce Core image downloads and show live backup, download, restart, and health-check progress during updates. diff --git a/Dockerfile b/Dockerfile index 8d5164e3..6526349c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -55,15 +55,15 @@ RUN apk add --no-cache ca-certificates tzdata # on container recreate. See go/cmd/ftw/main.go:66 — there # is no path resolution against the config file's directory; the open # call is literally state.Open(cfg.State.Path). -COPY --from=builder /out/ftw /app/ftw -COPY --from=builder /out/ftw-backup /app/ftw-backup -COPY drivers/ /app/drivers/ -COPY web/ /app/web/ +COPY --from=builder --chown=100:101 /out/ftw /app/ftw +COPY --from=builder --chown=100:101 /out/ftw-backup /app/ftw-backup +COPY --chown=100:101 drivers/ /app/drivers/ +COPY --chown=100:101 web/ /app/web/ COPY LICENSE NOTICE /usr/share/doc/ftw/ RUN ln -s /app/ftw /app/forty-two-watts && \ mkdir -p /app/data /app/data/drivers /run/ftw-update /run/ftw-optimizer && \ - chown -R 100:101 /app /run/ftw-update /run/ftw-optimizer + chown 100:101 /app/data /app/data/drivers /run/ftw-update /run/ftw-optimizer ENV HOME=/app/data diff --git a/go/cmd/ftw-updater/main.go b/go/cmd/ftw-updater/main.go index 5c33d74c..5cba2e9a 100644 --- a/go/cmd/ftw-updater/main.go +++ b/go/cmd/ftw-updater/main.go @@ -47,7 +47,7 @@ const ( // State mirrors selfupdate.UpdateStatus (we keep a local copy to avoid // importing the main module's internal package from this separate cmd). type State struct { - State string `json:"state"` // idle, pulling, restarting, restoring, done, failed + State string `json:"state"` // idle, pulling, restarting, checking, restoring, done, failed Action string `json:"action,omitempty"` // update, restart, rollback (#152) Component string `json:"component,omitempty"` Target string `json:"target,omitempty"` @@ -56,8 +56,14 @@ type State struct { SafetySnapshot string `json:"safety_snapshot,omitempty"` SafetyFiles []string `json:"safety_files,omitempty"` StartedAt time.Time `json:"started_at,omitempty"` + PhaseStartedAt time.Time `json:"phase_started_at,omitempty"` UpdatedAt time.Time `json:"updated_at,omitempty"` Message string `json:"message,omitempty"` + Step int `json:"step,omitempty"` + TotalSteps int `json:"total_steps,omitempty"` + ProgressCurrent int64 `json:"progress_current,omitempty"` + ProgressTotal int64 `json:"progress_total,omitempty"` + ProgressUnit string `json:"progress_unit,omitempty"` PreviousImageID string `json:"previous_image_id,omitempty"` PreviousImages map[string]string `json:"previous_images,omitempty"` } @@ -92,6 +98,10 @@ type server struct { // after an arbitrary N. Tests that exercise the "always-fail" path set // this to a small value to avoid looping forever. maxPullAttempts int + // statusHeartbeatInterval refreshes UpdatedAt while Docker performs a + // long pull, recreate, or health check. A zero value disables heartbeats + // in focused tests. + statusHeartbeatInterval time.Duration // runMu ensures only one pull+up runs at a time. HTTP handlers that // arrive while a job is in flight return 409. @@ -236,11 +246,12 @@ func main() { } srv := &server{ - composeFile: *compose, - statusPath: *statusPath, - skipPull: *skipPull, - pullRetryDelay: 60 * time.Second, - runner: dockerCompose, + composeFile: *compose, + statusPath: *statusPath, + skipPull: *skipPull, + pullRetryDelay: 60 * time.Second, + statusHeartbeatInterval: 2 * time.Second, + runner: dockerCompose, } // Auto-discover override files alongside the base, the same way the // compose CLI does when invoked without -f. Without this the sidecar @@ -456,7 +467,22 @@ func (s *server) runComponentJob(action, target, component string, startedAt tim return } } - s.writeState(State{State: "pulling", Action: action, Component: component, Target: target, StartedAt: now, UpdatedAt: now}) + totalSteps := 3 + pullStep := 1 + if action == "update" && spec.name == "core" { + totalSteps = 4 + pullStep = 2 + } + phaseState := func(state, message string, step int) State { + at := time.Now() + return State{ + State: state, Action: action, Component: component, Target: target, + StartedAt: now, PhaseStartedAt: at, UpdatedAt: at, + Message: message, Step: step, TotalSteps: totalSteps, + } + } + pullState := phaseState("pulling", "Downloading pinned release image", pullStep) + s.writeState(pullState) var env []string if target != "" { @@ -502,7 +528,12 @@ func (s *server) runComponentJob(action, target, component string, startedAt tim // single slow download on a 0.5 Mbps link (~400 MB image ≈ 90 min) // is never cut short by a shared outer deadline. attemptCtx, cancelAttempt := context.WithTimeout(context.Background(), 2*time.Hour) - pullErr = s.runner(attemptCtx, env, pullArgs...) + pullState.Message = fmt.Sprintf("Downloading pinned release image (attempt %d)", attempt) + pullState.UpdatedAt = time.Now() + s.writeState(pullState) + pullErr = s.runWithStateHeartbeat(pullState, func() error { + return s.runner(attemptCtx, env, pullArgs...) + }) cancelAttempt() if pullErr == nil { break @@ -511,7 +542,13 @@ func (s *server) runComponentJob(action, target, component string, startedAt tim if s.maxPullAttempts > 0 && attempt >= s.maxPullAttempts { break } - time.Sleep(s.pullRetryDelay) + pullState.Message = fmt.Sprintf("Download failed; retrying after %s", s.pullRetryDelay) + pullState.UpdatedAt = time.Now() + s.writeState(pullState) + _ = s.runWithStateHeartbeat(pullState, func() error { + time.Sleep(s.pullRetryDelay) + return nil + }) } if pullErr != nil { s.writeState(State{State: "failed", Action: action, Component: component, Target: target, StartedAt: now, UpdatedAt: time.Now(), Message: "pull failed: " + pullErr.Error()}) @@ -522,7 +559,8 @@ func (s *server) runComponentJob(action, target, component string, startedAt tim slog.Info("skip-pull active; continuing straight to compose up") } - s.writeState(State{State: "restarting", Action: action, Component: component, Target: target, StartedAt: now, UpdatedAt: time.Now()}) + restartState := phaseState("restarting", "Recreating service with the downloaded image", pullStep+1) + s.writeState(restartState) // compose up -d just recreates the container from an already-pulled // image — should complete in seconds, 10 min is a generous upper bound. @@ -536,15 +574,21 @@ func (s *server) runComponentJob(action, target, component string, startedAt tim // UI exposes as the "Restart" button. upArgs = s.composeArgs("up", "-d", "--force-recreate", spec.service) } - if err := s.runner(upCtx, env, upArgs...); err != nil { + if err := s.runWithStateHeartbeat(restartState, func() error { + return s.runner(upCtx, env, upArgs...) + }); err != nil { s.writeState(State{State: "failed", Action: action, Component: component, Target: target, StartedAt: now, UpdatedAt: time.Now(), Message: "up -d failed: " + err.Error()}) slog.Error("compose up failed", "err", err) return } if s.healthCheck != nil { + checkState := phaseState("checking", "Waiting for the new service to become ready", pullStep+2) + s.writeState(checkState) healthCtx, cancelHealth := context.WithTimeout(context.Background(), componentHealthTimeout(spec.name)) - healthErr := s.healthCheck(healthCtx, spec.service) + healthErr := s.runWithStateHeartbeat(checkState, func() error { + return s.healthCheck(healthCtx, spec.service) + }) cancelHealth() if healthErr != nil { if action == "update" && previousImageID != "" { @@ -563,7 +607,9 @@ func (s *server) runComponentJob(action, target, component string, startedAt tim // The main container is now being recreated. The brand-new replica // will read this "done" state on startup and serve it to the UI that's // still polling in the browser. - s.writeState(State{State: "done", Action: action, Component: component, Target: target, StartedAt: now, UpdatedAt: time.Now(), Message: "compose up -d completed", PreviousImageID: previousImageID}) + done := phaseState("done", "Update completed and service is ready", totalSteps) + done.PreviousImageID = previousImageID + s.writeState(done) } // requireHealthyOptimizer keeps a Core image without embedded Python from @@ -1060,6 +1106,32 @@ func (s *server) recoverRollbackSafety(ctx context.Context, base State, safetySn s.writeState(State{State: "failed", Action: base.Action, Snapshot: base.Snapshot, StartedAt: base.StartedAt, UpdatedAt: time.Now(), Message: message}) } +func (s *server) runWithStateHeartbeat(st State, run func() error) error { + if s.statusHeartbeatInterval <= 0 { + return run() + } + done := make(chan struct{}) + stopped := make(chan struct{}) + go func() { + defer close(stopped) + ticker := time.NewTicker(s.statusHeartbeatInterval) + defer ticker.Stop() + for { + select { + case <-ticker.C: + st.UpdatedAt = time.Now() + s.writeState(st) + case <-done: + return + } + } + }() + err := run() + close(done) + <-stopped + return err +} + func (s *server) writeState(st State) { s.stateMu.Lock() defer s.stateMu.Unlock() @@ -1164,7 +1236,7 @@ func (s *server) previousImageID(component string) string { // can restore the pre-rollback data before reporting failure. func (s *server) recoverCrashedState() { st := s.readState() - if st.State != "pulling" && st.State != "restarting" && st.State != "restoring" { + if st.State != "pulling" && st.State != "restarting" && st.State != "checking" && st.State != "restoring" { return } prev := st.State @@ -1234,14 +1306,19 @@ func (s *server) waitForServiceHealth(ctx context.Context, service string) error out, inspectErr := dockerOutput(ctx, "inspect", "--format", "{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}", containerID) if inspectErr == nil { switch status := strings.TrimSpace(out); status { - case "healthy", "running": + case "healthy", "running", "starting": if service != s.mainServiceName { + if status == "starting" { + break + } return nil } // Core intentionally answers /api/health while opening or // migrating a large database so Docker does not kill valid slow // boots. /api/status is served only by the fully wired API, so // require it before committing an update or restored state. + // Probe while Docker still reports "starting" so a healthy Core + // does not wait for the next 10-second healthcheck tick. if _, readyErr := dockerOutput(ctx, "exec", containerID, "wget", "-q", "-T", "4", "-O", "/dev/null", "http://127.0.0.1:8080/api/status"); readyErr == nil { return nil } diff --git a/go/cmd/ftw-updater/main_test.go b/go/cmd/ftw-updater/main_test.go index bb6d7cfc..2330034d 100644 --- a/go/cmd/ftw-updater/main_test.go +++ b/go/cmd/ftw-updater/main_test.go @@ -113,7 +113,10 @@ func TestSkipPull_BypassesPullStep(t *testing.T) { req := httptest.NewRequest(http.MethodPost, "/update", strings.NewReader(`{"action":"update","target":"v1.2.3"}`)) rr := httptest.NewRecorder() s.handleUpdate(rr, req) - waitForState(t, s, "done") + done := waitForState(t, s, "done") + if done.Step != 4 || done.TotalSteps != 4 || done.PhaseStartedAt.IsZero() { + t.Fatalf("done progress = %+v", done) + } calls := runner.snapshot() if len(calls) != 1 { t.Fatalf("skip-pull should yield 1 call (up only), got %d: %v", len(calls), calls) @@ -123,6 +126,27 @@ func TestSkipPull_BypassesPullStep(t *testing.T) { } } +func TestRunWithStateHeartbeatRefreshesLongPhase(t *testing.T) { + s, _ := newTestServer(t) + s.statusHeartbeatInterval = 5 * time.Millisecond + started := time.Now() + initial := State{ + State: "pulling", Action: "update", Component: "core", Target: "v1.2.3", + StartedAt: started, PhaseStartedAt: started, UpdatedAt: started, + Message: "Downloading pinned release image", Step: 2, TotalSteps: 4, + } + if err := s.runWithStateHeartbeat(initial, func() error { + time.Sleep(18 * time.Millisecond) + return nil + }); err != nil { + t.Fatal(err) + } + got := s.readState() + if !got.UpdatedAt.After(started) || got.State != "pulling" || got.Step != 2 || got.TotalSteps != 4 { + t.Fatalf("heartbeat state = %+v", got) + } +} + func TestOptimizerUpdateTargetsOnlyOptimizerService(t *testing.T) { s, runner := newTestServer(t) s.skipPull = true diff --git a/go/internal/api/api_selfupdate.go b/go/internal/api/api_selfupdate.go index d55aa06a..be183718 100644 --- a/go/internal/api/api_selfupdate.go +++ b/go/internal/api/api_selfupdate.go @@ -6,9 +6,11 @@ import ( "net/http" "path/filepath" "strings" + "sync" "time" "github.com/srcfl/ftw/go/internal/selfupdate" + "github.com/srcfl/ftw/go/internal/state" ) // handleVersionCheck returns the cached self-update state. ?force=1 bypasses @@ -71,7 +73,7 @@ func (s *Server) handleVersionChannel(w http.ResponseWriter, r *http.Request) { func versionUpdateInFlight(state string) bool { switch state { - case "starting", "snapshotting", "pulling", "restarting", "restoring": + case "starting", "snapshotting", "pulling", "restarting", "checking", "restoring": return true default: return false @@ -148,16 +150,21 @@ func (s *Server) handleVersionUpdate(w http.ResponseWriter, r *http.Request) { startedAt := time.Now() s.writeVersionUpdateStatus(selfupdate.UpdateStatus{ - State: "starting", - Action: "update", - Target: info.Latest, - StartedAt: startedAt, - UpdatedAt: time.Now(), - Message: "starting update", + State: "starting", + Action: "update", + Component: "core", + Target: info.Latest, + StartedAt: startedAt, + PhaseStartedAt: startedAt, + UpdatedAt: time.Now(), + Message: "starting update", + Step: 1, + TotalSteps: 4, }) s.recordComponentStatus(selfupdate.UpdateStatus{ State: "starting", Action: "update", Component: "core", Target: info.Latest, - StartedAt: startedAt, UpdatedAt: startedAt, Message: "starting update", + StartedAt: startedAt, PhaseStartedAt: startedAt, UpdatedAt: startedAt, + Message: "starting update", Step: 1, TotalSteps: 4, }, info.Current) go s.runVersionUpdate(startedAt, info.Current, info.Latest) @@ -172,21 +179,37 @@ func (s *Server) handleVersionUpdate(w http.ResponseWriter, r *http.Request) { func (s *Server) runVersionUpdate(startedAt time.Time, current, latest string) { defer s.versionUpdateMu.Unlock() - writeUpdateStatus := func(state, message string) { + writeUpdateStatus := func(updateState, message string) { + now := time.Now() s.writeVersionUpdateStatus(selfupdate.UpdateStatus{ - State: state, - Action: "update", - Target: latest, - StartedAt: startedAt, - UpdatedAt: time.Now(), - Message: message, + State: updateState, + Action: "update", + Component: "core", + Target: latest, + StartedAt: startedAt, + PhaseStartedAt: now, + UpdatedAt: now, + Message: message, + TotalSteps: 4, }) } snapshotSkipped := s.deps.SnapshotDir == "" if !snapshotSkipped { - writeUpdateStatus("snapshotting", "creating backup snapshot") - if _, err := s.createPreUpdateSnapshot("update", current, latest); err != nil { + phaseStarted := time.Now() + status := selfupdate.UpdateStatus{ + State: "snapshotting", Action: "update", Component: "core", Target: latest, + StartedAt: startedAt, PhaseStartedAt: phaseStarted, UpdatedAt: phaseStarted, + Message: "Copying full history database", Step: 1, TotalSteps: 4, + } + s.writeVersionUpdateStatus(status) + heartbeat := newUpdateStatusHeartbeat(s.deps.SelfUpdate, status) + heartbeat.Start() + _, err := s.createPreUpdateSnapshotWithProgress("update", current, latest, func(progress state.BackupProgress) { + heartbeat.SetBackupProgress(progress) + }) + heartbeat.Stop() + if err != nil { writeUpdateStatus("failed", "snapshot failed: "+err.Error()) return } @@ -200,6 +223,80 @@ func (s *Server) runVersionUpdate(startedAt time.Time, current, latest string) { } } +type updateStatusHeartbeat struct { + checker *selfupdate.Checker + + mu sync.Mutex + status selfupdate.UpdateStatus + phase string + stop chan struct{} + done chan struct{} +} + +func newUpdateStatusHeartbeat(checker *selfupdate.Checker, status selfupdate.UpdateStatus) *updateStatusHeartbeat { + return &updateStatusHeartbeat{ + checker: checker, + status: status, + stop: make(chan struct{}), + done: make(chan struct{}), + } +} + +func (h *updateStatusHeartbeat) Start() { + go func() { + defer close(h.done) + ticker := time.NewTicker(2 * time.Second) + defer ticker.Stop() + for { + select { + case <-ticker.C: + h.publish(nil) + case <-h.stop: + return + } + } + }() +} + +func (h *updateStatusHeartbeat) Stop() { + close(h.stop) + <-h.done +} + +func (h *updateStatusHeartbeat) SetBackupProgress(progress state.BackupProgress) { + h.publish(func(status *selfupdate.UpdateStatus) { + if progress.Phase != h.phase { + h.phase = progress.Phase + status.PhaseStartedAt = time.Now() + } + status.ProgressCurrent = progress.CompletedBytes + status.ProgressTotal = progress.TotalBytes + status.ProgressUnit = "" + switch progress.Phase { + case state.BackupPhaseCopying: + status.Message = "Copying full history database" + case state.BackupPhaseCompressing: + status.Message = "Compressing rollback backup" + status.ProgressUnit = "bytes" + case state.BackupPhaseSyncing: + status.Message = "Syncing rollback backup to disk" + status.ProgressUnit = "bytes" + } + }) +} + +func (h *updateStatusHeartbeat) publish(update func(*selfupdate.UpdateStatus)) { + h.mu.Lock() + defer h.mu.Unlock() + if update != nil { + update(&h.status) + } + h.status.UpdatedAt = time.Now() + if err := h.checker.WriteStatus(h.status); err != nil { + slog.Warn("selfupdate: backup progress write failed", "err", err) + } +} + func (s *Server) writeVersionUpdateStatus(st selfupdate.UpdateStatus) { if s.deps.SelfUpdate == nil { return diff --git a/go/internal/api/api_selfupdate_test.go b/go/internal/api/api_selfupdate_test.go index 7a2e6c46..fe84a746 100644 --- a/go/internal/api/api_selfupdate_test.go +++ b/go/internal/api/api_selfupdate_test.go @@ -326,6 +326,33 @@ func TestVersionUpdate_CreatesSnapshotBeforeTrigger(t *testing.T) { } } +func TestUpdateStatusHeartbeatPublishesBackupProgress(t *testing.T) { + path := filepath.Join(t.TempDir(), "state.json") + checker := selfupdate.New(selfupdate.Config{StatusPath: path}, nil) + started := time.Now().Add(-time.Minute) + heartbeat := newUpdateStatusHeartbeat(checker, selfupdate.UpdateStatus{ + State: "snapshotting", Action: "update", Component: "core", Target: "v1.5.0", + StartedAt: started, PhaseStartedAt: started, Step: 1, TotalSteps: 4, + }) + heartbeat.Start() + heartbeat.SetBackupProgress(state.BackupProgress{ + Phase: state.BackupPhaseCompressing, CompletedBytes: 25, TotalBytes: 100, + }) + heartbeat.Stop() + + got := checker.Status() + if got.State != "snapshotting" || got.Step != 1 || got.TotalSteps != 4 { + t.Fatalf("status = %+v", got) + } + if got.Message != "Compressing rollback backup" || + got.ProgressCurrent != 25 || got.ProgressTotal != 100 || got.ProgressUnit != "bytes" { + t.Fatalf("backup progress = %+v", got) + } + if got.PhaseStartedAt.IsZero() || got.UpdatedAt.IsZero() { + t.Fatalf("progress timestamps = %+v", got) + } +} + func gunzipTestFile(t *testing.T, src, dst string) { t.Helper() in, err := os.Open(src) diff --git a/go/internal/api/snapshots.go b/go/internal/api/snapshots.go index 80a6d93b..9b705d86 100644 --- a/go/internal/api/snapshots.go +++ b/go/internal/api/snapshots.go @@ -11,6 +11,8 @@ import ( "sort" "strings" "time" + + "github.com/srcfl/ftw/go/internal/state" ) // snapshotKeepCount caps the normal update snapshots we retain. A rollback @@ -62,6 +64,13 @@ type SnapshotInfo struct { // safety-netted update, so if we can't produce the safety net we'd // rather refuse the update than silently proceed without one. func (s *Server) createPreUpdateSnapshot(action, fromVersion, toVersion string) (SnapshotInfo, error) { + return s.createPreUpdateSnapshotWithProgress(action, fromVersion, toVersion, nil) +} + +func (s *Server) createPreUpdateSnapshotWithProgress( + action, fromVersion, toVersion string, + report func(state.BackupProgress), +) (SnapshotInfo, error) { if s.deps.SnapshotDir == "" { return SnapshotInfo{}, errors.New("snapshot dir not configured") } @@ -93,7 +102,7 @@ func (s *Server) createPreUpdateSnapshot(action, fromVersion, toVersion string) // 1. A complete state.db via VACUUM INTO + gzip. Rollback backups must // include history and samples; the compact daily corruption-recovery // snapshot deliberately excludes those large tables and is not safe here. - if err := s.deps.State.BackupToCompressed(filepath.Join(dir, "state.db.gz")); err != nil { + if err := s.deps.State.BackupToCompressedWithProgress(filepath.Join(dir, "state.db.gz"), report); err != nil { return SnapshotInfo{}, fmt.Errorf("state snapshot: %w", err) } captured = append(captured, "state.db.gz") diff --git a/go/internal/selfupdate/selfupdate.go b/go/internal/selfupdate/selfupdate.go index 2b24f82e..be127c74 100644 --- a/go/internal/selfupdate/selfupdate.go +++ b/go/internal/selfupdate/selfupdate.go @@ -66,6 +66,10 @@ const ( // state file hasn't been refreshed within this window. Catches the // sidecar crashing mid-pull so the UI overlay can unblock. staleThreshold = 5 * time.Minute + // Sidecars released before phase heartbeats can stay silent for the + // full Docker pull. Their pull timeout is two hours, so do not turn a + // slow but valid download into a false failure before then. + legacySidecarStaleThreshold = 2 * time.Hour ) // Channel controls which immutable release stream the checker follows. @@ -175,14 +179,20 @@ const MaxReleaseBodyBytes = 16 * 1024 // through unchanged. The main service may also write early states before // handing off to the sidecar, e.g. starting/snapshotting. type UpdateStatus struct { - State string `json:"state"` // idle, starting, snapshotting, pulling, restarting, restoring, done, failed + State string `json:"state"` // idle, starting, snapshotting, pulling, restarting, checking, restoring, done, failed Action string `json:"action,omitempty"` Component string `json:"component,omitempty"` Target string `json:"target,omitempty"` Snapshot string `json:"snapshot,omitempty"` StartedAt time.Time `json:"started_at,omitempty"` + PhaseStartedAt time.Time `json:"phase_started_at,omitempty"` UpdatedAt time.Time `json:"updated_at,omitempty"` Message string `json:"message,omitempty"` + Step int `json:"step,omitempty"` + TotalSteps int `json:"total_steps,omitempty"` + ProgressCurrent int64 `json:"progress_current,omitempty"` + ProgressTotal int64 `json:"progress_total,omitempty"` + ProgressUnit string `json:"progress_unit,omitempty"` PreviousImageID string `json:"previous_image_id,omitempty"` PreviousImages map[string]string `json:"previous_images,omitempty"` } @@ -708,8 +718,8 @@ func (c *Checker) postSidecar(ctx context.Context, body []byte) error { } // Status reads the sidecar's state.json. Missing or unreadable returns -// {state: idle}. A pulling/restarting state whose last heartbeat is older -// than staleThreshold is surfaced as failed so the UI overlay unblocks. +// {state: idle}. An in-flight state whose last heartbeat is too old is +// surfaced as failed so the UI overlay unblocks. func (c *Checker) Status() UpdateStatus { if c.cfg.StatusPath == "" { return UpdateStatus{State: "idle"} @@ -724,10 +734,11 @@ func (c *Checker) Status() UpdateStatus { return UpdateStatus{State: "idle"} } if isInFlightState(st.State) && !st.UpdatedAt.IsZero() { - if c.cfg.Now().Sub(st.UpdatedAt) > staleThreshold { + threshold := updateStatusStaleThreshold(st) + if c.cfg.Now().Sub(st.UpdatedAt) > threshold { st.State = "failed" if st.Message == "" { - st.Message = "no heartbeat from updater in 5 min" + st.Message = fmt.Sprintf("no updater heartbeat for %s", threshold) } } } @@ -785,13 +796,23 @@ func (c *Checker) WriteStatus(st UpdateStatus) error { func isInFlightState(state string) bool { switch state { - case "starting", "snapshotting", "pulling", "restarting", "restoring": + case "starting", "snapshotting", "pulling", "restarting", "checking", "restoring": return true default: return false } } +func updateStatusStaleThreshold(st UpdateStatus) time.Duration { + if st.PhaseStartedAt.IsZero() { + switch st.State { + case "pulling", "restarting", "restoring": + return legacySidecarStaleThreshold + } + } + return staleThreshold +} + func inferChannel(version string) Channel { switch { case strings.HasPrefix(version, "edge-"): diff --git a/go/internal/selfupdate/selfupdate_test.go b/go/internal/selfupdate/selfupdate_test.go index 1ea37119..4e81c872 100644 --- a/go/internal/selfupdate/selfupdate_test.go +++ b/go/internal/selfupdate/selfupdate_test.go @@ -526,11 +526,45 @@ func TestStatus_ReadsAndDetectsStale(t *testing.T) { t.Errorf("fresh status = %+v, want restoring snapshot-123", s) } - stale := UpdateStatus{State: "pulling", Action: "update", UpdatedAt: time.Now().Add(-10 * time.Minute)} + stale := UpdateStatus{ + State: "pulling", + Action: "update", + PhaseStartedAt: time.Now().Add(-10 * time.Minute), + UpdatedAt: time.Now().Add(-10 * time.Minute), + } writeJSON(t, path, stale) if s := c.Status(); s.State != "failed" { t.Errorf("stale state = %q, want failed", s.State) } + stale.State = "checking" + writeJSON(t, path, stale) + if s := c.Status(); s.State != "failed" { + t.Errorf("stale checking state = %q, want failed", s.State) + } +} + +func TestStatus_AllowsSilentLegacyPullUntilItsDockerTimeout(t *testing.T) { + path := filepath.Join(t.TempDir(), "state.json") + now := time.Now() + c := New(Config{StatusPath: path, Now: func() time.Time { return now }}, nil) + + writeJSON(t, path, UpdateStatus{ + State: "pulling", + Action: "update", + UpdatedAt: now.Add(-15 * time.Minute), + }) + if got := c.Status(); got.State != "pulling" { + t.Fatalf("15-minute legacy pull state = %q, want pulling", got.State) + } + + writeJSON(t, path, UpdateStatus{ + State: "pulling", + Action: "update", + UpdatedAt: now.Add(-legacySidecarStaleThreshold - time.Second), + }) + if got := c.Status(); got.State != "failed" { + t.Fatalf("expired legacy pull state = %q, want failed", got.State) + } } func TestWriteStatusPublishesPreSidecarState(t *testing.T) { @@ -540,11 +574,17 @@ func TestWriteStatusPublishesPreSidecarState(t *testing.T) { started := time.Now().Add(-time.Second) if err := c.WriteStatus(UpdateStatus{ - State: "snapshotting", - Action: "update", - Target: "v1.5.0", - StartedAt: started, - Message: "creating backup snapshot", + State: "snapshotting", + Action: "update", + Target: "v1.5.0", + StartedAt: started, + PhaseStartedAt: started, + Message: "creating backup snapshot", + Step: 1, + TotalSteps: 4, + ProgressCurrent: 50, + ProgressTotal: 100, + ProgressUnit: "bytes", }); err != nil { t.Fatalf("write status: %v", err) } @@ -556,6 +596,10 @@ func TestWriteStatusPublishesPreSidecarState(t *testing.T) { if got.UpdatedAt.IsZero() { t.Fatal("UpdatedAt should be filled") } + if got.Step != 1 || got.TotalSteps != 4 || got.ProgressCurrent != 50 || + got.ProgressTotal != 100 || got.ProgressUnit != "bytes" || got.PhaseStartedAt.IsZero() { + t.Fatalf("progress fields = %+v", got) + } } func TestWriteStatusPreservesPerComponentRollbackHistory(t *testing.T) { diff --git a/go/internal/state/store.go b/go/internal/state/store.go index 1e1667d2..af17d02b 100644 --- a/go/internal/state/store.go +++ b/go/internal/state/store.go @@ -370,6 +370,20 @@ func (s *Store) SnapshotTo(dstPath string) error { return nil } +// BackupProgress reports the current phase of a complete database backup. +// TotalBytes is known once SQLite has produced the consistent raw copy. +type BackupProgress struct { + Phase string + CompletedBytes int64 + TotalBytes int64 +} + +const ( + BackupPhaseCopying = "copying_database" + BackupPhaseCompressing = "compressing_database" + BackupPhaseSyncing = "syncing_backup" +) + // BackupToCompressed writes a complete, point-in-time copy of state.db as a // gzip stream. Unlike SnapshotTo, this is a user-data backup: it includes the // history and sample tables as well as configuration, models, and identities. @@ -380,6 +394,13 @@ func (s *Store) SnapshotTo(dstPath string) error { // SQLite cannot stream VACUUM INTO, so the complete raw copy is materialised // next to dstPath, compressed, synced, and removed. dstPath must not exist. func (s *Store) BackupToCompressed(dstPath string) error { + return s.BackupToCompressedWithProgress(dstPath, nil) +} + +// BackupToCompressedWithProgress is BackupToCompressed with phase and byte +// progress. The callback may take long enough to write a small status file, +// but it must not call back into Store. +func (s *Store) BackupToCompressedWithProgress(dstPath string, report func(BackupProgress)) error { if s == nil || s.db == nil { return fmt.Errorf("store: backup on nil store") } @@ -392,6 +413,7 @@ func (s *Store) BackupToCompressed(dstPath string) error { rawPath := dstPath + ".raw.tmp" _ = os.Remove(rawPath) defer os.Remove(rawPath) + reportBackupProgress(report, BackupProgress{Phase: BackupPhaseCopying}) escaped := strings.ReplaceAll(rawPath, "'", "''") if _, err := s.db.Exec(fmt.Sprintf("VACUUM INTO '%s'", escaped)); err != nil { return fmt.Errorf("backup to %s: %w", rawPath, err) @@ -402,6 +424,15 @@ func (s *Store) BackupToCompressed(dstPath string) error { return fmt.Errorf("open backup temp: %w", err) } defer in.Close() + rawInfo, err := in.Stat() + if err != nil { + return fmt.Errorf("stat backup temp: %w", err) + } + rawBytes := rawInfo.Size() + reportBackupProgress(report, BackupProgress{ + Phase: BackupPhaseCompressing, + TotalBytes: rawBytes, + }) out, err := os.OpenFile(dstPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) if err != nil { return fmt.Errorf("create compressed backup: %w", err) @@ -418,13 +449,24 @@ func (s *Store) BackupToCompressed(dstPath string) error { if err != nil { return fmt.Errorf("create gzip writer: %w", err) } - if _, err := io.Copy(zw, in); err != nil { + progress := &backupProgressReader{ + r: in, + total: rawBytes, + report: report, + lastAt: time.Now(), + } + if _, err := io.CopyBuffer(zw, progress, make([]byte, 1<<20)); err != nil { _ = zw.Close() return fmt.Errorf("compress backup: %w", err) } if err := zw.Close(); err != nil { return fmt.Errorf("finish compressed backup: %w", err) } + reportBackupProgress(report, BackupProgress{ + Phase: BackupPhaseSyncing, + CompletedBytes: rawBytes, + TotalBytes: rawBytes, + }) if err := out.Sync(); err != nil { return fmt.Errorf("sync compressed backup: %w", err) } @@ -435,6 +477,35 @@ func (s *Store) BackupToCompressed(dstPath string) error { return nil } +type backupProgressReader struct { + r io.Reader + total int64 + completed int64 + report func(BackupProgress) + lastAt time.Time +} + +func (r *backupProgressReader) Read(p []byte) (int, error) { + n, err := r.r.Read(p) + r.completed += int64(n) + now := time.Now() + if r.completed == r.total || now.Sub(r.lastAt) >= time.Second { + reportBackupProgress(r.report, BackupProgress{ + Phase: BackupPhaseCompressing, + CompletedBytes: r.completed, + TotalBytes: r.total, + }) + r.lastAt = now + } + return n, err +} + +func reportBackupProgress(report func(BackupProgress), progress BackupProgress) { + if report != nil { + report(progress) + } +} + type snapshotSchemaRow struct { objType string name string diff --git a/go/internal/state/store_test.go b/go/internal/state/store_test.go index ba8d1f28..e08ac5fe 100644 --- a/go/internal/state/store_test.go +++ b/go/internal/state/store_test.go @@ -806,5 +806,36 @@ func TestBackupToCompressedPreservesCompleteHistory(t *testing.T) { } } +func TestBackupToCompressedReportsPhasesAndBytes(t *testing.T) { + s := freshStore(t) + dst := filepath.Join(t.TempDir(), "state.db.gz") + var progress []BackupProgress + if err := s.BackupToCompressedWithProgress(dst, func(update BackupProgress) { + progress = append(progress, update) + }); err != nil { + t.Fatal(err) + } + if len(progress) < 4 { + t.Fatalf("progress = %#v", progress) + } + if progress[0].Phase != BackupPhaseCopying { + t.Fatalf("first progress = %#v", progress[0]) + } + var compressed, synced bool + for _, update := range progress { + if update.Phase == BackupPhaseCompressing && update.TotalBytes > 0 && + update.CompletedBytes == update.TotalBytes { + compressed = true + } + if update.Phase == BackupPhaseSyncing && update.TotalBytes > 0 && + update.CompletedBytes == update.TotalBytes { + synced = true + } + } + if !compressed || !synced { + t.Fatalf("progress did not finish compression and sync: %#v", progress) + } +} + // avoid unused import if context not used var _ = context.Canceled diff --git a/scripts/test-container-boundaries.sh b/scripts/test-container-boundaries.sh index c11aa393..ea827f6b 100755 --- a/scripts/test-container-boundaries.sh +++ b/scripts/test-container-boundaries.sh @@ -13,6 +13,11 @@ grep -q '^FROM alpine:' Dockerfile grep -q '^COPY optimizer/' Dockerfile.optimizer grep -q '/out/ftw-backup' Dockerfile grep -q '/app/ftw-backup' Dockerfile +grep -q -- '--chown=100:101 /out/ftw' Dockerfile +if grep -q 'chown -R 100:101 /app' Dockerfile; then + echo "Dockerfile must set ownership while copying; a full-tree chown duplicates every app layer" >&2 + exit 1 +fi grep -q '^ ftw-optimizer:' docker-compose.yml grep -q 'FTW_OPTIMIZER_SOCKET: /run/ftw-optimizer/optimizer.sock' docker-compose.yml diff --git a/web/components/ftw-update-check.js b/web/components/ftw-update-check.js index bea9c384..43cb9064 100644 --- a/web/components/ftw-update-check.js +++ b/web/components/ftw-update-check.js @@ -17,9 +17,8 @@ // button that can only fail. // 4. Update-now posts /api/version/update, opens an -based // progress overlay, polls /api/version/update/status, and -// cache-busts reloads on `done`. Failure and ~3-minute timeout -// swap the spinner for a Reload / Continue-setup escape hatch so -// the operator can bail out. +// cache-busts reloads on `done`. Long phases keep polling while the +// UI offers a Reload / Continue-setup escape hatch. // 5. Continue-anyway hides the card for this page load only. We do // NOT POST /api/version/skip — that would silence the dashboard's // too, which is a separate decision the operator @@ -37,6 +36,7 @@ import "./ftw-modal.js"; const STATUS_POLL_MS = 2000; const UPDATE_SOFT_TIMEOUT_MS = 180 * 1000; +const SNAPSHOT_SOFT_TIMEOUT_MS = 15 * 60 * 1000; class FtwUpdateCheck extends FtwElement { static styles = ` @@ -152,6 +152,24 @@ class FtwUpdateCheck extends FtwElement { color: var(--fg-dim); margin: 0; } + .update-progress { + width: min(340px, 78vw); + height: 8px; + margin: 0.2rem auto 0.7rem; + overflow: hidden; + border: 1px solid var(--line); + border-radius: 999px; + background: color-mix(in srgb, var(--fg) 5%, transparent); + } + .update-progress > span { + display: block; + height: 100%; + min-width: 4px; + border-radius: inherit; + background: var(--accent-e); + transition: width 0.25s ease; + } + .update-step { margin: 0.25rem 0; font-weight: 600; } @keyframes spin { to { transform: rotate(360deg); } } /* Hide the modal's X during an active update. The operator should @@ -264,7 +282,10 @@ class FtwUpdateCheck extends FtwElement { // should have rejected the promise above — but guard anyway so a // late resolution can never trigger _reload() after bailout. if (!st || this._phase !== "updating") return; - this._status = st; + const keepTimeout = this._status && this._status.timed_out && + this._status.state === st.state && + (this._status.phase_started_at || "") === (st.phase_started_at || ""); + this._status = keepTimeout ? Object.assign({}, st, { timed_out: true }) : st; if (st.state === "failed") { this._fail(st.message || "Update failed"); return; @@ -281,12 +302,16 @@ class FtwUpdateCheck extends FtwElement { }) .catch(() => { /* aborted, or main container mid-restart — both fine */ }); - if (Date.now() - this._updateStartedAt > UPDATE_SOFT_TIMEOUT_MS) { - if (this._phase === "updating") { - this._stopPolling(); - this._phase = "timedOut"; - this.update(); - } + const phaseStarted = this._status && this._status.phase_started_at + ? Date.parse(this._status.phase_started_at) + : 0; + const timeoutStartedAt = phaseStarted > 0 ? phaseStarted : this._updateStartedAt; + const timeout = this._status && this._status.state === "snapshotting" + ? SNAPSHOT_SOFT_TIMEOUT_MS + : UPDATE_SOFT_TIMEOUT_MS; + if (Date.now() - timeoutStartedAt > timeout && this._phase === "updating") { + this._status = Object.assign({}, this._status, { timed_out: true }); + this.update(); } } @@ -389,7 +414,18 @@ class FtwUpdateCheck extends FtwElement { const st = this._status || { state: "starting" }; const busy = this._phase === "updating"; const failed = this._phase === "failed"; - const timedOut = this._phase === "timedOut"; + const timedOut = !!st.timed_out || this._phase === "timedOut"; + const progress = operationProgress(st); + const elapsed = Math.max(0, Math.round((Date.now() - this._updateStartedAt) / 1000)); + const phaseStarted = st.phase_started_at ? Date.parse(st.phase_started_at) : 0; + const phaseElapsed = Math.max(0, Math.round((Date.now() - (phaseStarted > 0 ? phaseStarted : this._updateStartedAt)) / 1000)); + const progressHTML = ` +
+ +
+

Step ${progress.step} of ${progress.total} · ${escapeHTML(stateLabel(st.state))}

+

${escapeHTML(st.message || "")}

+

This step: ${escapeHTML(formatElapsed(phaseElapsed))} · Total: ${escapeHTML(formatElapsed(elapsed))}

`; let title = "Updating"; let msg = stateLabel(st.state) + "…"; @@ -406,7 +442,6 @@ class FtwUpdateCheck extends FtwElement { `; } else if (timedOut) { - const elapsed = Math.round((Date.now() - this._updateStartedAt) / 1000); title = "Taking longer than expected"; msg = `Still working after ${elapsed}s. You can reload to check, or continue setup and let the update finish in the background.`; hint = ""; @@ -423,7 +458,7 @@ class FtwUpdateCheck extends FtwElement { ${escapeHTML(title)}
${busy ? `` : ""} -

${escapeHTML(msg)}

+ ${busy && !failed ? progressHTML : `

${escapeHTML(msg)}

`} ${hint ? `

${escapeHTML(hint)}

` : ""}
${actions} @@ -455,12 +490,37 @@ function escapeHTML(s) { function stateLabel(state) { switch (state) { + case "snapshotting": return "Creating backup"; case "pulling": return "Pulling new image"; case "restarting": return "Applying update"; + case "checking": return "Checking service health"; case "done": return "Reloading"; case "failed": return "Failed"; default: return "Starting update"; } } +function operationProgress(st) { + let total = Number(st && st.total_steps) || 4; + let step = Number(st && st.step) || 0; + if (!step) { + switch (st && st.state) { + case "snapshotting": step = 1; break; + case "pulling": step = 2; break; + case "restarting": step = 3; break; + case "checking": + case "done": step = 4; break; + default: step = 1; + } + } + step = Math.max(0, Math.min(step, total)); + return { step, total, percent: Math.round((step / total) * 100) }; +} + +function formatElapsed(seconds) { + const value = Math.max(0, Number(seconds) || 0); + if (value < 60) return `${value}s`; + return `${Math.floor(value / 60)}m ${value % 60}s`; +} + customElements.define("ftw-update-check", FtwUpdateCheck); diff --git a/web/update-badge.js b/web/update-badge.js index 5af0ddad..7d4e43a1 100644 --- a/web/update-badge.js +++ b/web/update-badge.js @@ -58,6 +58,7 @@ } connectedCallback() { + this._resumeUpdateStatus(); this._refresh(false); this._refreshComponents(false); this._refreshDriverCatalog(); @@ -68,6 +69,28 @@ }, CHECK_INTERVAL_MS); } + _resumeUpdateStatus() { + apiFetch("/api/version/update/status", { cache: "no-store" }) + .then((r) => (r.ok ? r.json() : null)) + .then((st) => { + if (!st || !isUpdateInFlight(st.state) || this._phase === "updating") return; + const started = st.started_at ? Date.parse(st.started_at) : 0; + this._phase = "updating"; + this._sidecarState = st; + this._updateStartedAt = started > 0 ? started : Date.now(); + this._expectedRun = { + action: st.action || "update", + target: st.target || "", + snapshot: st.snapshot || "", + component: st.component || "", + }; + this._render(); + this._startElapsedTicker(); + this._startStatusPolling(); + }) + .catch(() => { /* the service may still be starting */ }); + } + disconnectedCallback() { clearInterval(this._checkTimer); clearInterval(this._statusTimer); @@ -517,7 +540,10 @@ .then((r) => (r.ok ? r.json() : null)) .then((st) => { if (st && this._statusMatchesCurrentRun(st)) { - this._sidecarState = st; + const keepTimeout = this._sidecarState && this._sidecarState.timedOut && + this._sidecarState.state === st.state && + (this._sidecarState.phase_started_at || "") === (st.phase_started_at || ""); + this._sidecarState = keepTimeout ? Object.assign({}, st, { timedOut: true }) : st; this._render(); if (st.state === "done") { this._attemptReload(); @@ -555,7 +581,11 @@ const timeoutMs = this._sidecarState && this._sidecarState.state === "snapshotting" ? SNAPSHOT_SOFT_TIMEOUT_MS : UPDATE_SOFT_TIMEOUT_MS; - if (Date.now() - this._updateStartedAt <= timeoutMs) return false; + const phaseStarted = this._sidecarState && this._sidecarState.phase_started_at + ? Date.parse(this._sidecarState.phase_started_at) + : 0; + const timeoutStartedAt = phaseStarted > 0 ? phaseStarted : this._updateStartedAt; + if (Date.now() - timeoutStartedAt <= timeoutMs) return false; if (!this._sidecarState || this._sidecarState.state === "done" || this._sidecarState.timedOut) return false; this._sidecarState = Object.assign({}, this._sidecarState, { timedOut: true }); return true; @@ -904,6 +934,19 @@ const spinner = st.state === "failed" ? "" : ``; const timedOut = !!st.timedOut; const failed = st.state === "failed"; + const progress = operationProgress(st, action); + const phaseStarted = st.phase_started_at ? Date.parse(st.phase_started_at) : 0; + const phaseElapsed = Math.max(0, Math.round((Date.now() - (phaseStarted > 0 ? phaseStarted : this._updateStartedAt)) / 1000)); + const byteProgress = st.progress_unit === "bytes" && st.progress_total > 0 + ? `

${escapeHTML(formatBytes(st.progress_current || 0))} / ${escapeHTML(formatBytes(st.progress_total))}

` + : ""; + const progressHTML = failed ? "" : ` +
+ +
+

Step ${progress.step} of ${progress.total} · ${escapeHTML(label)}

+

This step: ${escapeHTML(formatElapsed(phaseElapsed))}

+ ${byteProgress}`; const body = failed ? `

${escapeHTML(st.message || "Update failed")}

@@ -911,9 +954,9 @@ : timedOut ? `

Still working after ${elapsed}s. The main container may have been slow to restart.

You can reload manually if the UI keeps the overlay stuck.

` - : `

${escapeHTML(label)}…

+ : `${progressHTML} ${this._operationDetailHTML(st)} -

Elapsed: ${elapsed}s. The page will reload automatically.

`; +

Total: ${escapeHTML(formatElapsed(elapsed))}. The page will reload automatically.

`; const footer = failed || timedOut ? ` @@ -953,6 +996,8 @@ return msg + `

Downloading the pinned release image from GHCR.

`; case "restarting": return msg + `

Recreating the service. Short polling errors are expected while the container swaps.

`; + case "checking": + return msg + `

Core has started. Waiting for its API and health checks before marking the update complete.

`; case "restoring": return msg + `

Restoring files from the selected backup snapshot.

`; default: @@ -1398,6 +1443,25 @@ animation: spin 0.9s linear infinite; margin-bottom: 0.6rem; } + .update-progress { + width: min(360px, 82vw); + height: 8px; + margin: 0.2rem auto 0.7rem; + overflow: hidden; + border: 1px solid var(--line, #334155); + border-radius: 999px; + background: rgba(255,255,255,0.04); + } + .update-progress > span { + display: block; + height: 100%; + min-width: 4px; + border-radius: inherit; + background: var(--accent-e, #f59e0b); + transition: width 0.25s ease; + } + .update-step { margin: 0.25rem 0; font-weight: 600; } + .body.center .dim { margin: 0.25rem 0; } @keyframes spin { to { transform: rotate(360deg); } } `; } @@ -1412,6 +1476,7 @@ if (action === "restart") return "Restarting service"; if (action === "rollback") return "Restarting on restored state"; return "Applying update"; + case "checking": return "Checking service health"; case "done": return "Reloading"; case "failed": return "Failed"; default: @@ -1421,6 +1486,50 @@ } } + function isUpdateInFlight(state) { + return ["starting", "snapshotting", "pulling", "restarting", "checking", "restoring"].includes(state); + } + + function operationProgress(st, action) { + let total = Number(st && st.total_steps) || 0; + let step = Number(st && st.step) || 0; + if (!total) { + const hasBackup = action === "update" && (!st || !st.component || st.component === "core"); + total = hasBackup ? 4 : 3; + const offset = hasBackup ? 1 : 0; + switch (st && st.state) { + case "snapshotting": step = 1; break; + case "pulling": step = 1 + offset; break; + case "restarting": step = 2 + offset; break; + case "checking": + case "done": step = 3 + offset; break; + default: step = 1; + } + } + step = Math.max(0, Math.min(step, total)); + return { step, total, percent: total > 0 ? Math.round((step / total) * 100) : 0 }; + } + + function formatElapsed(seconds) { + const value = Math.max(0, Number(seconds) || 0); + if (value < 60) return `${value}s`; + const minutes = Math.floor(value / 60); + const rest = value % 60; + return `${minutes}m ${rest}s`; + } + + function formatBytes(bytes) { + let value = Math.max(0, Number(bytes) || 0); + const units = ["B", "KB", "MB", "GB"]; + let unit = 0; + while (value >= 1024 && unit < units.length - 1) { + value /= 1024; + unit++; + } + const digits = unit > 0 && value < 10 ? 1 : 0; + return `${value.toFixed(digits)} ${units[unit]}`; + } + // safeHref rejects anything that isn't http:/https:. The release-notes URL // comes from the GitHub Releases API, but we belt-and-brace here: an // attacker who somehow lands a javascript:/data: URL into the payload diff --git a/web/update-progress.test.mjs b/web/update-progress.test.mjs new file mode 100644 index 00000000..5cf915bb --- /dev/null +++ b/web/update-progress.test.mjs @@ -0,0 +1,32 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import test from "node:test"; + +const badge = readFileSync(new URL("./update-badge.js", import.meta.url), "utf8"); +const setup = readFileSync(new URL("./components/ftw-update-check.js", import.meta.url), "utf8"); +const dockerfile = readFileSync(new URL("../Dockerfile", import.meta.url), "utf8"); + +test("update UI resumes work and shows each server phase", () => { + assert.match(badge, /_resumeUpdateStatus\(\)/); + assert.match(badge, /phase_started_at/); + assert.match(badge, /progress_current/); + assert.match(badge, /progress_total/); + assert.match(badge, /case "checking":\s+return "Checking service health"/); + assert.match(badge, /This step:/); + assert.match(badge, /Total:/); +}); + +test("setup keeps polling when a safe update takes longer", () => { + assert.match(setup, /SNAPSHOT_SOFT_TIMEOUT_MS = 15 \* 60 \* 1000/); + assert.match(setup, /timed_out: true/); + assert.doesNotMatch(setup, /this\._stopPolling\(\);\s+this\._phase = "timedOut"/); + assert.match(setup, /case "snapshotting": return "Creating backup"/); + assert.match(setup, /case "checking":\s+return "Checking service health"/); +}); + +test("Core image sets ownership during copy without a duplicate app layer", () => { + assert.match(dockerfile, /COPY --from=builder --chown=100:101 \/out\/ftw\s+\/app\/ftw/); + assert.match(dockerfile, /COPY --chown=100:101 drivers\/\s+\/app\/drivers\//); + assert.match(dockerfile, /COPY --chown=100:101 web\/\s+\/app\/web\//); + assert.doesNotMatch(dockerfile, /chown -R 100:101 \/app/); +}); From 16465c0b66d1bf2ab32868d3f0876f554046fc24 Mon Sep 17 00:00:00 2001 From: Fredrik Ahlgren Date: Thu, 23 Jul 2026 08:57:27 +0200 Subject: [PATCH 2/2] perf: skip redundant update backups --- .changeset/faster-visible-updates.md | 2 +- .github/workflows/beta.yml | 9 ++++ .github/workflows/release.yml | 6 +++ go/cmd/ftw/main.go | 7 +-- go/internal/api/api_selfupdate.go | 20 +++++--- go/internal/api/api_selfupdate_test.go | 60 +++++++++++++++++++--- go/internal/selfupdate/selfupdate.go | 62 ++++++++++++++++++++--- go/internal/selfupdate/selfupdate_test.go | 44 ++++++++++++++++ go/internal/state/store.go | 4 ++ go/internal/state/store_test.go | 17 +++++++ state-schema.json | 3 ++ web/release-metadata.test.mjs | 18 +++++++ web/update-badge.js | 7 ++- web/update-progress.test.mjs | 1 + 14 files changed, 235 insertions(+), 25 deletions(-) create mode 100644 state-schema.json diff --git a/.changeset/faster-visible-updates.md b/.changeset/faster-visible-updates.md index 610e0c02..dc205194 100644 --- a/.changeset/faster-visible-updates.md +++ b/.changeset/faster-visible-updates.md @@ -2,4 +2,4 @@ "ftw": patch --- -Reduce Core image downloads and show live backup, download, restart, and health-check progress during updates. +Skip full history backups when the release uses the same database schema, reduce Core image downloads, and show live backup, download, restart, and health-check progress. diff --git a/.github/workflows/beta.yml b/.github/workflows/beta.yml index aedf4a84..51bcece0 100644 --- a/.github/workflows/beta.yml +++ b/.github/workflows/beta.yml @@ -22,6 +22,7 @@ jobs: runs-on: ubuntu-latest outputs: version: ${{ steps.version.outputs.version }} + state_schema: ${{ steps.version.outputs.state_schema }} steps: - name: Checkout selected commit uses: actions/checkout@v5 @@ -45,6 +46,11 @@ jobs: echo "${TAG} does not match package.json version ${PACKAGE_VERSION}" >&2 exit 1 fi + STATE_SCHEMA="$(node -p "require('./state-schema.json').version")" + if [[ ! "${STATE_SCHEMA}" =~ ^[1-9][0-9]*$ ]]; then + echo "state-schema.json must contain a positive integer version" >&2 + exit 1 + fi git config user.name "github-actions[bot]" git config user.email "41898282+github-actions[bot]@users.noreply.github.com" @@ -60,6 +66,7 @@ jobs: git push origin "refs/tags/${TAG}" fi echo "version=${TAG}" >> "${GITHUB_OUTPUT}" + echo "state_schema=${STATE_SCHEMA}" >> "${GITHUB_OUTPUT}" docker: name: beta docker (${{ matrix.target }}) @@ -149,6 +156,7 @@ jobs: env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} TAG: ${{ needs.tag.outputs.version }} + STATE_SCHEMA: ${{ needs.tag.outputs.state_schema }} run: | set -euo pipefail if gh release view "${TAG}" --repo "${GITHUB_REPOSITORY}" >/dev/null 2>&1; then @@ -159,4 +167,5 @@ jobs: --repo "${GITHUB_REPOSITORY}" \ --title "${TAG}" \ --prerelease \ + --notes "" \ --generate-notes diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d8fca77b..90e68cf3 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -231,6 +231,12 @@ jobs: # and the next version-shaped heading) and prepends the # Hitchhiker codename header. node scripts/apply-codename.cjs "${VERSION}" > release-notes.md + STATE_SCHEMA="$(node -p "require('./state-schema.json').version")" + if [[ ! "${STATE_SCHEMA}" =~ ^[1-9][0-9]*$ ]]; then + echo "state-schema.json must contain a positive integer version" >&2 + exit 1 + fi + printf '\n\n\n' "${STATE_SCHEMA}" >> release-notes.md echo "--- release-notes.md ---" cat release-notes.md echo "--- end ---" diff --git a/go/cmd/ftw/main.go b/go/cmd/ftw/main.go index 34ba779d..1dc14a2e 100644 --- a/go/cmd/ftw/main.go +++ b/go/cmd/ftw/main.go @@ -1811,9 +1811,10 @@ func main() { "env", "FTW_SELFUPDATE_CURRENT_VERSION") } selfUpdater = selfupdate.New(selfupdate.Config{ - CurrentVersion: current, - SocketPath: envOr("FTW_UPDATER_SOCKET", "/run/ftw-update/sock"), - StatusPath: envOr("FTW_UPDATER_STATUS", "/run/ftw-update/state.json"), + CurrentVersion: current, + CurrentStateSchema: state.SchemaVersion, + SocketPath: envOr("FTW_UPDATER_SOCKET", "/run/ftw-update/sock"), + StatusPath: envOr("FTW_UPDATER_STATUS", "/run/ftw-update/state.json"), // Publish events.UpdateAvailable when a new release lands so // the notifications service (or any other subscriber) can act // without polling the checker directly. diff --git a/go/internal/api/api_selfupdate.go b/go/internal/api/api_selfupdate.go index be183718..0c395b83 100644 --- a/go/internal/api/api_selfupdate.go +++ b/go/internal/api/api_selfupdate.go @@ -149,6 +149,11 @@ func (s *Server) handleVersionUpdate(w http.ResponseWriter, r *http.Request) { } startedAt := time.Now() + fullBackupRequired := info.FullBackupRequired + startMessage := "starting update" + if !fullBackupRequired { + startMessage = "Database schema unchanged; full history backup not needed" + } s.writeVersionUpdateStatus(selfupdate.UpdateStatus{ State: "starting", Action: "update", @@ -157,26 +162,29 @@ func (s *Server) handleVersionUpdate(w http.ResponseWriter, r *http.Request) { StartedAt: startedAt, PhaseStartedAt: startedAt, UpdatedAt: time.Now(), - Message: "starting update", + Message: startMessage, Step: 1, TotalSteps: 4, }) s.recordComponentStatus(selfupdate.UpdateStatus{ State: "starting", Action: "update", Component: "core", Target: info.Latest, StartedAt: startedAt, PhaseStartedAt: startedAt, UpdatedAt: startedAt, - Message: "starting update", Step: 1, TotalSteps: 4, + Message: startMessage, Step: 1, TotalSteps: 4, }, info.Current) - go s.runVersionUpdate(startedAt, info.Current, info.Latest) + go s.runVersionUpdate(startedAt, info.Current, info.Latest, fullBackupRequired) resp := map[string]any{"status": "started", "action": "update", "target": info.Latest} - if s.deps.SnapshotDir == "" { + if !fullBackupRequired { + resp["snapshot_skipped"] = true + resp["snapshot_skip_reason"] = "database schema unchanged" + } else if s.deps.SnapshotDir == "" { resp["snapshot_skipped"] = true } writeJSON(w, 202, resp) } -func (s *Server) runVersionUpdate(startedAt time.Time, current, latest string) { +func (s *Server) runVersionUpdate(startedAt time.Time, current, latest string, fullBackupRequired bool) { defer s.versionUpdateMu.Unlock() writeUpdateStatus := func(updateState, message string) { @@ -194,7 +202,7 @@ func (s *Server) runVersionUpdate(startedAt time.Time, current, latest string) { }) } - snapshotSkipped := s.deps.SnapshotDir == "" + snapshotSkipped := s.deps.SnapshotDir == "" || !fullBackupRequired if !snapshotSkipped { phaseStarted := time.Now() status := selfupdate.UpdateStatus{ diff --git a/go/internal/api/api_selfupdate_test.go b/go/internal/api/api_selfupdate_test.go index fe84a746..d348b8ea 100644 --- a/go/internal/api/api_selfupdate_test.go +++ b/go/internal/api/api_selfupdate_test.go @@ -54,6 +54,14 @@ func newCheckerAgainstWithStatus(t *testing.T, tag, current, statusPath string) } func newCheckerAgainstWithStatusAndSocket(t *testing.T, tag, current, statusPath, socketPath string) *selfupdate.Checker { + return newCheckerAgainstOptions(t, tag, current, statusPath, socketPath, "", 0) +} + +func newCheckerAgainstOptions( + t *testing.T, + tag, current, statusPath, socketPath, releaseBody string, + currentStateSchema int, +) *selfupdate.Checker { t.Helper() const repo = "srcfl/ftw" @@ -74,19 +82,21 @@ func newCheckerAgainstWithStatusAndSocket(t *testing.T, tag, current, statusPath _ = json.NewEncoder(w).Encode(map[string]any{ "tag_name": tag, "html_url": "https://example/releases/" + tag, + "body": releaseBody, "published_at": time.Now().Format(time.RFC3339), }) })) t.Cleanup(relSrv.Close) c := selfupdate.New(selfupdate.Config{ - Repo: repo, - CurrentVersion: current, - RegistryBaseURL: regSrv.URL, - LatestReleaseURL: relSrv.URL, - CheckInterval: time.Hour, - SocketPath: socketPath, - StatusPath: statusPath, + Repo: repo, + CurrentVersion: current, + CurrentStateSchema: currentStateSchema, + RegistryBaseURL: regSrv.URL, + LatestReleaseURL: relSrv.URL, + CheckInterval: time.Hour, + SocketPath: socketPath, + StatusPath: statusPath, }, newMemStore()) if _, err := c.Check(t.Context(), true); err != nil { t.Fatalf("priming check: %v", err) @@ -326,6 +336,42 @@ func TestVersionUpdate_CreatesSnapshotBeforeTrigger(t *testing.T) { } } +func TestVersionUpdateSkipsFullHistoryBackupWhenSchemaIsUnchanged(t *testing.T) { + dir := t.TempDir() + statusPath := filepath.Join(dir, "update-state.json") + socketPath := startFakeSidecar(t, http.StatusInternalServerError) + c := newCheckerAgainstOptions( + t, "v1.5.0", "v1.4.0", statusPath, socketPath, + "", 1, + ) + st, err := state.Open(filepath.Join(dir, "state.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { st.Close() }) + snapDir := filepath.Join(dir, "snapshots") + srv := New(&Deps{SelfUpdate: c, State: st, SnapshotDir: snapDir}) + + req := httptest.NewRequest(http.MethodPost, "/api/version/update", nil) + rr := httptest.NewRecorder() + srv.Handler().ServeHTTP(rr, req) + if rr.Code != http.StatusAccepted { + t.Fatalf("status = %d body=%s", rr.Code, rr.Body.String()) + } + var response map[string]any + if err := json.Unmarshal(rr.Body.Bytes(), &response); err != nil { + t.Fatal(err) + } + if response["snapshot_skipped"] != true || + response["snapshot_skip_reason"] != "database schema unchanged" { + t.Fatalf("response = %+v", response) + } + waitUntil(t, func() bool { return c.Status().State == "failed" }) + if _, err := os.Stat(snapDir); !os.IsNotExist(err) { + t.Fatalf("schema-compatible update created snapshot dir: %v", err) + } +} + func TestUpdateStatusHeartbeatPublishesBackupProgress(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") checker := selfupdate.New(selfupdate.Config{StatusPath: path}, nil) diff --git a/go/internal/selfupdate/selfupdate.go b/go/internal/selfupdate/selfupdate.go index be127c74..5b9f316a 100644 --- a/go/internal/selfupdate/selfupdate.go +++ b/go/internal/selfupdate/selfupdate.go @@ -116,6 +116,10 @@ type Config struct { // CurrentVersion is the running binary's version (from main.Version). CurrentVersion string + // CurrentStateSchema is the running Core's on-disk state format. Core + // releases publish the target value in their release notes. A missing or + // different target keeps the full pre-update backup fail-closed. + CurrentStateSchema int // CheckInterval is the probe cadence. 0 = 1 h. CheckInterval time.Duration // SocketPath is where the sidecar listens. Empty disables Trigger. @@ -154,12 +158,15 @@ type Info struct { // operators can read what's about to be applied without opening // a new tab. Capped at MaxReleaseBodyBytes to keep a pathological // release note from ballooning the Info payload. - ReleaseBody string `json:"release_body,omitempty"` - CheckedAt time.Time `json:"checked_at,omitempty"` - UpdateAvailable bool `json:"update_available"` - Skipped bool `json:"skipped"` - SkippedVersion string `json:"skipped_version,omitempty"` - Err string `json:"err,omitempty"` + ReleaseBody string `json:"release_body,omitempty"` + CurrentStateSchema int `json:"current_state_schema,omitempty"` + TargetStateSchema int `json:"target_state_schema,omitempty"` + FullBackupRequired bool `json:"full_backup_required"` + CheckedAt time.Time `json:"checked_at,omitempty"` + UpdateAvailable bool `json:"update_available"` + Skipped bool `json:"skipped"` + SkippedVersion string `json:"skipped_version,omitempty"` + Err string `json:"err,omitempty"` // SidecarReady is true only when the ftw-updater sidecar's Unix socket // is present at SocketPath — i.e. the full pull+restart flow is wired // up, which in practice means a docker-compose deploy. Native / WSL @@ -254,6 +261,8 @@ func New(cfg Config, store Store) *Checker { } c := &Checker{cfg: cfg, store: store, skippedKey: componentSkippedKey, channelKey: componentChannelKey} c.info.Current = cfg.CurrentVersion + c.info.CurrentStateSchema = cfg.CurrentStateSchema + c.info.FullBackupRequired = true c.info.Channel = channel c.info.Channels = append([]Channel(nil), availableChannels...) c.mu.Lock() @@ -337,10 +346,15 @@ func (c *Checker) Check(ctx context.Context, force bool) (Info, error) { return info, nil } if deployable { + targetStateSchema := releaseStateSchema(rel.Body) c.info.Latest = targetTag c.info.PublishedAt = rel.PublishedAt c.info.ReleaseNotesURL = rel.HtmlURL - c.info.ReleaseBody = truncateBody(rel.Body) + c.info.ReleaseBody = truncateBody(releaseBodyWithoutStateSchema(rel.Body)) + c.info.TargetStateSchema = targetStateSchema + c.info.FullBackupRequired = c.cfg.CurrentStateSchema <= 0 || + targetStateSchema <= 0 || + targetStateSchema != c.cfg.CurrentStateSchema c.info.UpdateAvailable = channelUpdateAvailable(targetTag, c.info.Current) } else { // Either GH has no published release yet, or the build workflow @@ -558,6 +572,8 @@ func (c *Checker) SetChannel(channel Channel) error { c.info.PublishedAt = time.Time{} c.info.ReleaseNotesURL = "" c.info.ReleaseBody = "" + c.info.TargetStateSchema = 0 + c.info.FullBackupRequired = true c.info.CheckedAt = time.Time{} c.info.UpdateAvailable = false c.info.Err = "" @@ -957,3 +973,35 @@ func truncateBody(b string) string { } return b[:MaxReleaseBodyBytes] + "\n\n…(truncated — see release notes for full changelog)" } + +func releaseStateSchema(body string) int { + const prefix = "") + if end < 0 { + return 0 + } + schema, err := strconv.Atoi(strings.TrimSpace(value[:end])) + if err != nil || schema <= 0 { + return 0 + } + return schema +} + +func releaseBodyWithoutStateSchema(body string) string { + const prefix = "") + if end < 0 { + return body + } + return strings.TrimSpace(body[:start] + rest[end+3:]) +} diff --git a/go/internal/selfupdate/selfupdate_test.go b/go/internal/selfupdate/selfupdate_test.go index 4e81c872..5dea5b45 100644 --- a/go/internal/selfupdate/selfupdate_test.go +++ b/go/internal/selfupdate/selfupdate_test.go @@ -150,6 +150,50 @@ func TestCheck_UpdateAvailable(t *testing.T) { } } +func TestCheckRequiresBackupOnlyWhenReleaseSchemaIsMissingOrDifferent(t *testing.T) { + const repo = "srcfl/ftw" + for _, tc := range []struct { + name string + body string + targetSchema int + backupRequired bool + }{ + {name: "same", body: "", targetSchema: 7, backupRequired: false}, + {name: "different", body: "", targetSchema: 8, backupRequired: true}, + {name: "missing", body: "ordinary release notes", targetSchema: 0, backupRequired: true}, + {name: "invalid", body: "", targetSchema: 0, backupRequired: true}, + } { + t.Run(tc.name, func(t *testing.T) { + reg := newFakeRegistry(t, repo) + reg.addTag("v1.3.0") + registry := reg.server() + defer registry.Close() + releases := fakeReleasesServer(t, fakeRelease{ + tag: "v1.3.0", body: tc.body, published: time.Now(), + }) + defer releases.Close() + + c := New(Config{ + Repo: repo, CurrentVersion: "v1.2.0", CurrentStateSchema: 7, + RegistryBaseURL: registry.URL, LatestReleaseURL: releases.URL, + }, newMemStore()) + info, err := c.Check(t.Context(), true) + if err != nil { + t.Fatal(err) + } + if info.CurrentStateSchema != 7 || info.TargetStateSchema != tc.targetSchema { + t.Fatalf("schema info = %+v", info) + } + if info.FullBackupRequired != tc.backupRequired { + t.Fatalf("FullBackupRequired = %v, want %v", info.FullBackupRequired, tc.backupRequired) + } + if strings.Contains(info.ReleaseBody, "ftw-state-schema") { + t.Fatalf("internal schema marker leaked into release notes: %q", info.ReleaseBody) + } + }) + } +} + func TestPrefixedOptimizerBootstrapIsMonotonicFromLegacyBeta(t *testing.T) { const image = "srcfl/ftw-optimizer" reg := newFakeRegistry(t, image) diff --git a/go/internal/state/store.go b/go/internal/state/store.go index af17d02b..e9a19917 100644 --- a/go/internal/state/store.go +++ b/go/internal/state/store.go @@ -25,6 +25,10 @@ import ( ) const ( + // SchemaVersion identifies the on-disk state format for update rollback. + // Increase it before a release that cannot safely reopen the same state.db + // with the prior Core version. + SchemaVersion = 1 // HotRetention = 30 days at 5s resolution HotRetention = 30 * 24 * time.Hour // WarmRetention = 12 months at 15-min buckets diff --git a/go/internal/state/store_test.go b/go/internal/state/store_test.go index e08ac5fe..0c5089c5 100644 --- a/go/internal/state/store_test.go +++ b/go/internal/state/store_test.go @@ -3,6 +3,7 @@ package state import ( "compress/gzip" "context" + "encoding/json" "io" "os" "path/filepath" @@ -10,6 +11,22 @@ import ( "time" ) +func TestSchemaVersionMatchesReleaseMetadata(t *testing.T) { + raw, err := os.ReadFile(filepath.Join("..", "..", "..", "state-schema.json")) + if err != nil { + t.Fatal(err) + } + var metadata struct { + Version int `json:"version"` + } + if err := json.Unmarshal(raw, &metadata); err != nil { + t.Fatal(err) + } + if metadata.Version != SchemaVersion { + t.Fatalf("state-schema.json version = %d, Go schema version = %d", metadata.Version, SchemaVersion) + } +} + func freshStore(t *testing.T) *Store { t.Helper() path := filepath.Join(t.TempDir(), "state.db") diff --git a/state-schema.json b/state-schema.json new file mode 100644 index 00000000..61a2092b --- /dev/null +++ b/state-schema.json @@ -0,0 +1,3 @@ +{ + "version": 1 +} diff --git a/web/release-metadata.test.mjs b/web/release-metadata.test.mjs index e05eba03..4518e275 100644 --- a/web/release-metadata.test.mjs +++ b/web/release-metadata.test.mjs @@ -14,6 +14,13 @@ const releaseWorkflow = readFileSync( join(repoRoot, ".github", "workflows", "release.yml"), "utf8", ); +const betaWorkflow = readFileSync( + join(repoRoot, ".github", "workflows", "beta.yml"), + "utf8", +); +const stateSchema = JSON.parse( + readFileSync(join(repoRoot, "state-schema.json"), "utf8"), +); const changesetCheckWorkflow = readFileSync( join(repoRoot, ".github", "workflows", "changeset-check.yml"), "utf8", @@ -85,6 +92,17 @@ describe("release metadata", () => { assert.match(releaseWorkflow, /version:\s+npm run version-packages/); }); + it("publishes the state schema in beta and stable release notes", () => { + assert.ok(Number.isInteger(stateSchema.version) && stateSchema.version > 0); + assert.match(betaWorkflow, /require\('\.\/state-schema\.json'\)\.version/); + assert.match(betaWorkflow, /--notes ""/); + assert.match(releaseWorkflow, /require\('\.\/state-schema\.json'\)\.version/); + assert.match( + releaseWorkflow, + /\\n' "\$\{STATE_SCHEMA\}" >> release-notes\.md/, + ); + }); + it("validates Changesets against the fetched PR base", () => { assert.match( changesetCheckWorkflow, diff --git a/web/update-badge.js b/web/update-badge.js index 7d4e43a1..fda217fb 100644 --- a/web/update-badge.js +++ b/web/update-badge.js @@ -495,11 +495,16 @@ } if (action === "update") { const skipped = resp.body && resp.body.snapshot_skipped; + const skipReason = resp.body && resp.body.snapshot_skip_reason; this._sidecarState = { state: skipped ? "pulling" : "snapshotting", action, target: this._expectedRun.target, - message: skipped ? "backup snapshot skipped for this update" : "creating backup snapshot", + message: skipped + ? (skipReason === "database schema unchanged" + ? "Database schema unchanged; full history backup not needed" + : "Backup snapshot skipped for this update") + : "Creating backup snapshot", }; this._render(); } diff --git a/web/update-progress.test.mjs b/web/update-progress.test.mjs index 5cf915bb..a31d8b24 100644 --- a/web/update-progress.test.mjs +++ b/web/update-progress.test.mjs @@ -14,6 +14,7 @@ test("update UI resumes work and shows each server phase", () => { assert.match(badge, /case "checking":\s+return "Checking service health"/); assert.match(badge, /This step:/); assert.match(badge, /Total:/); + assert.match(badge, /Database schema unchanged; full history backup not needed/); }); test("setup keeps polling when a safe update takes longer", () => {