From 8ad1db90e29c8277be9bc8859b30df75a320faeb Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sat, 24 Oct 2026 16:12:35 +0100 Subject: [PATCH 1/7] test(desktop-gpui): record the video ids the mock Cap server creates and deletes --- apps/desktop-gpui/src/upload/verify_tests.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/apps/desktop-gpui/src/upload/verify_tests.rs b/apps/desktop-gpui/src/upload/verify_tests.rs index 7d13956beb..503978eeb2 100644 --- a/apps/desktop-gpui/src/upload/verify_tests.rs +++ b/apps/desktop-gpui/src/upload/verify_tests.rs @@ -207,6 +207,8 @@ struct MockState { next_id: u32, confirmations: Vec, deletes: u32, + created: Vec, + deleted: Vec, problems: Vec, } @@ -433,6 +435,16 @@ impl MockCap { self.lock().confirmations.clone() } + /// Video ids the mock created, in order. + pub(super) fn created(&self) -> Vec { + self.lock().created.clone() + } + + /// Video ids the mock was asked to delete, in order. + pub(super) fn deleted(&self) -> Vec { + self.lock().deleted.clone() + } + pub(super) fn deletes(&self) -> u32 { self.lock().deletes } @@ -792,6 +804,7 @@ impl Server { state.next_id += 1; format!("verifyvideo{:04}", state.next_id) }); + state.created.push(id.clone()); reply(200, json!({ "id": id })) } ("POST", "/api/upload/multipart/initiate") => { @@ -924,6 +937,9 @@ impl Server { } ("DELETE", "/api/desktop/video/delete") => { state.deletes += 1; + if let Some(id) = query.get("videoId") { + state.deleted.push(id.clone()); + } reply(200, json!({ "success": true })) } _ => { From cbecef56d04328f51b78eef373993658618cb1ca Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sat, 24 Oct 2026 16:12:35 +0100 Subject: [PATCH 2/7] test(desktop-gpui): drive the real app through Instant crashes, cancels and recovery against the mock server --- apps/desktop-gpui/src/upload.rs | 2 + .../desktop-gpui/src/upload/real_app_tests.rs | 398 ++++++++++++++++++ 2 files changed, 400 insertions(+) create mode 100644 apps/desktop-gpui/src/upload/real_app_tests.rs diff --git a/apps/desktop-gpui/src/upload.rs b/apps/desktop-gpui/src/upload.rs index 53878a1977..fb65253319 100644 --- a/apps/desktop-gpui/src/upload.rs +++ b/apps/desktop-gpui/src/upload.rs @@ -20,6 +20,8 @@ pub(crate) mod instant_cleanup; mod multipart_tests; pub(crate) mod queue; #[cfg(test)] +mod real_app_tests; +#[cfg(test)] mod verify_tests; const MIN_CHUNK_SIZE: u64 = 5 * 1024 * 1024; diff --git a/apps/desktop-gpui/src/upload/real_app_tests.rs b/apps/desktop-gpui/src/upload/real_app_tests.rs new file mode 100644 index 0000000000..07734585ee --- /dev/null +++ b/apps/desktop-gpui/src/upload/real_app_tests.rs @@ -0,0 +1,398 @@ +//! The real app against `MockCap`: record, crash, cancel and recover Instant +//! recordings, then check what reached the mock server and what was left on +//! disk. Each case needs a built app and screen recording permission, so they +//! are ignored by default: +//! +//! ```sh +//! CAP_GPUI_REAL_APP_BIN=target/debug/cap-gpui \ +//! cargo test --bin cap-gpui real_app -- --ignored --test-threads=1 +//! ``` + +use super::verify_tests::{API_KEY, MockCap, OWNER}; +use serde_json::{Value, json}; +use std::{ + path::{Path, PathBuf}, + process::{Child, Command}, + time::{Duration, Instant, SystemTime}, +}; + +const JOURNAL: &str = "gpui-instant-cleanup.json"; +const STATE: &str = "instant-upload.json"; + +struct Sandbox { + root: tempfile::TempDir, +} + +impl Sandbox { + /// Kept under `CAP_GPUI_REAL_APP_ARTIFACTS` when that is set, so a failed + /// run leaves the app's logs and recordings behind. + fn new(server: &str, countdown: u32) -> Self { + let root = match std::env::var_os("CAP_GPUI_REAL_APP_ARTIFACTS") { + Some(directory) => { + std::fs::create_dir_all(&directory).unwrap(); + tempfile::Builder::new() + .prefix("sandbox-") + .disable_cleanup(true) + .tempdir_in(directory) + .unwrap() + } + None => tempfile::tempdir().unwrap(), + }; + std::fs::create_dir_all(root.path().join("recordings")).unwrap(); + std::fs::create_dir_all(root.path().join("logs")).unwrap(); + std::fs::write( + root.path().join("store"), + serde_json::to_vec(&json!({ + "general_settings": { + "serverUrl": server, + "enableNotifications": false, + "hasCompletedOnboarding": true, + "recordingCountdown": countdown, + "enableTelemetry": false, + }, + "recording_start_safety": { + "confirmBeforeRecordingWithoutMicrophone": false, + }, + "auth": { + "secret": { "api_key": API_KEY }, + "user_id": OWNER, + "plan": { "upgraded": true, "manual": true }, + "organizations": [], + }, + })) + .unwrap(), + ) + .unwrap(); + Self { root } + } + + fn launch(&self, name: &str, env: &[(&str, &str)]) -> Child { + let binary = std::env::var("CAP_GPUI_REAL_APP_BIN") + .expect("set CAP_GPUI_REAL_APP_BIN to the built cap-gpui binary"); + let log = std::fs::File::create(self.root.path().join(format!("{name}.log"))).unwrap(); + let mut command = Command::new(binary); + command + .env("CAP_GPUI_APP_DATA_DIR", self.root.path()) + .env("CAP_GPUI_RECORDINGS_DIR", self.recordings()) + .env("CAP_GPUI_LOG_DIR", self.root.path().join("logs")) + .env("CAP_GPUI_NO_TRAY", "1") + .env("CAP_GPUI_NO_MENUS", "1") + .env("CAP_GPUI_AUTO_NO_MIC", "1") + .env("RUST_LOG", "cap_gpui=info,cap_recording=info") + .stdout(log.try_clone().unwrap()) + .stderr(log); + for (key, value) in env { + command.env(key, value); + } + command.spawn().unwrap() + } + + fn log(&self, name: &str) -> String { + std::fs::read_to_string(self.root.path().join(format!("{name}.log"))).unwrap_or_default() + } + + fn recordings(&self) -> PathBuf { + self.root.path().join("recordings") + } + + fn journal(&self) -> Vec { + std::fs::read(self.root.path().join(JOURNAL)) + .ok() + .and_then(|bytes| serde_json::from_slice::(&bytes).ok()) + .and_then(|journal| journal["entries"].as_array().cloned()) + .unwrap_or_default() + } + + fn projects(&self) -> Vec { + let mut projects: Vec<_> = std::fs::read_dir(self.recordings()) + .map(|entries| { + entries + .flatten() + .map(|entry| entry.path()) + .filter(|path| path.extension().is_some_and(|extension| extension == "cap")) + .collect() + }) + .unwrap_or_default(); + projects.sort(); + projects + } + + fn project_for(&self, video_id: &str) -> Option { + self.projects() + .into_iter() + .find(|project| state(project).is_some_and(|state| state["video_id"] == video_id)) + } + + /// Moves every journal entry and upload state an hour into the past, as + /// if the crashed session ended long before the relaunch. + fn age(&self) { + let path = self.root.path().join(JOURNAL); + if let Ok(bytes) = std::fs::read(&path) { + let mut journal: Value = serde_json::from_slice(&bytes).unwrap(); + let past = chrono::Utc::now().timestamp() - 3600; + for entry in journal["entries"].as_array_mut().into_iter().flatten() { + entry["created_at"] = json!(past); + entry["next_attempt_at"] = json!(past); + } + std::fs::write(&path, serde_json::to_vec(&journal).unwrap()).unwrap(); + } + for project in self.projects() { + let state = project.join(STATE); + if state.exists() { + std::fs::File::options() + .write(true) + .open(&state) + .unwrap() + .set_modified(SystemTime::now() - Duration::from_secs(3600)) + .unwrap(); + } + } + } +} + +fn state(project: &Path) -> Option { + serde_json::from_slice(&std::fs::read(project.join(STATE)).ok()?).ok() +} + +fn meta(project: &Path) -> Option { + serde_json::from_slice(&std::fs::read(project.join("recording-meta.json")).ok()?).ok() +} + +fn media_files(project: &Path) -> usize { + fn walk(directory: &Path, depth: usize) -> usize { + let Ok(entries) = std::fs::read_dir(directory) else { + return 0; + }; + entries + .flatten() + .map(|entry| { + let path = entry.path(); + let name = entry.file_name().to_string_lossy().to_ascii_lowercase(); + if path.is_dir() { + if depth > 0 { walk(&path, depth - 1) } else { 0 } + } else { + let sized = entry.metadata().is_ok_and(|metadata| metadata.len() > 0); + usize::from( + sized + && (name.ends_with(".m4s") + || (name.ends_with(".mp4") && !name.starts_with("init"))), + ) + } + }) + .sum() + } + walk(&project.join("content"), 3) +} + +fn finished_segments(project: &Path) -> usize { + std::fs::read(project.join("content/display/manifest.json")) + .ok() + .and_then(|bytes| serde_json::from_slice::(&bytes).ok()) + .and_then(|manifest| { + manifest["segments"].as_array().map(|segments| { + segments + .iter() + .filter(|segment| segment["is_complete"] == true) + .count() + }) + }) + .unwrap_or(0) +} + +/// Waits for `done` while the app runs. If the app does not get there in +/// time, its threads are sampled into the sandbox before the test fails, so a +/// hang can be diagnosed from the artifacts. +async fn wait_for_app( + sandbox: &Sandbox, + app: &Child, + what: &str, + timeout: Duration, + mut done: impl FnMut() -> bool, +) { + let deadline = Instant::now() + timeout; + while !done() { + if Instant::now() >= deadline { + let report = sandbox.root.path().join("hang-sample.txt"); + #[cfg(target_os = "macos")] + let _ = Command::new("sample") + .args([&app.id().to_string(), "5", "-file"]) + .arg(&report) + .status(); + panic!( + "timed out waiting for {what}; app {} sampled to {}", + app.id(), + report.display() + ); + } + tokio::time::sleep(Duration::from_millis(50)).await; + } +} + +fn kill(mut app: Child) { + let _ = app.kill(); + let _ = app.wait(); +} + +/// A normal Instant recording uploads, verifies, and leaves nothing for the +/// cleanup journal. +#[ignore] +#[tokio::test(flavor = "multi_thread")] +async fn real_app_instant_recording_uploads_and_verifies() { + let mock = MockCap::start(Vec::new()).await; + let sandbox = Sandbox::new(&mock.url, 0); + let app = sandbox.launch("record", &[("CAP_GPUI_AUTO_RECORD", "instant:6")]); + wait_for_app( + &sandbox, + &app, + "a verified upload", + Duration::from_secs(180), + || { + sandbox + .projects() + .first() + .and_then(|project| state(project)) + .is_some_and(|state| state["phase"] == "Verified") + }, + ) + .await; + kill(app); + let video = mock.created(); + assert_eq!(video.len(), 1, "{video:?}"); + assert!( + mock.confirmations() + .iter() + .any(|confirmation| confirmation["videoId"] == video[0].as_str()) + ); + assert!(mock.deleted().is_empty(), "{:?}", mock.deleted()); + assert!(sandbox.journal().is_empty(), "{:?}", sandbox.journal()); +} + +/// Kills the app at points from the server video's creation through the +/// first seconds of capture, then relaunches long after. No video whose +/// recording holds media is ever deleted, and every crashed recording that +/// holds none and has its folder is deleted. +#[ignore] +#[tokio::test(flavor = "multi_thread")] +async fn real_app_crashes_never_delete_a_recording_with_media() { + let mock = MockCap::start(Vec::new()).await; + let sandbox = Sandbox::new(&mock.url, 2); + for (attempt, offset_ms) in [0u64, 150, 400, 800, 1500, 2500, 4000, 7000] + .into_iter() + .enumerate() + { + let created = mock.created().len(); + let app = sandbox.launch( + &format!("crash-{attempt}"), + &[("CAP_GPUI_AUTO_RECORD", "instant:60")], + ); + wait_for_app( + &sandbox, + &app, + "the server video", + Duration::from_secs(60), + || mock.created().len() > created, + ) + .await; + tokio::time::sleep(Duration::from_millis(offset_ms)).await; + kill(app); + } + sandbox.age(); + let app = sandbox.launch("sweep", &[]); + tokio::time::sleep(Duration::from_secs(20)).await; + kill(app); + + let deleted = mock.deleted(); + for video in mock.created() { + let project = sandbox.project_for(&video); + let media = project.as_deref().map(media_files).unwrap_or(0); + if media > 0 { + assert!( + !deleted.contains(&video), + "{video} was deleted although its recording holds media" + ); + } else if let Some(project) = project + && meta(&project).is_some_and(|meta| meta.get("recording").is_some()) + { + assert!( + deleted.contains(&video), + "{video} crashed before capturing anything but was kept" + ); + } + } +} + +/// Kills the app mid-recording with media saved, then relaunches and presses +/// Recover: the recovered recording uploads under the same video id. +#[ignore] +#[tokio::test(flavor = "multi_thread")] +async fn real_app_recovered_recording_uploads() { + let mock = MockCap::start(Vec::new()).await; + let sandbox = Sandbox::new(&mock.url, 0); + let app = sandbox.launch("record", &[("CAP_GPUI_AUTO_RECORD", "instant:120")]); + wait_for_app( + &sandbox, + &app, + "four finished segments", + Duration::from_secs(120), + || { + sandbox + .projects() + .first() + .is_some_and(|project| finished_segments(project) >= 4) + }, + ) + .await; + kill(app); + let project = sandbox.projects().remove(0); + let video = state(&project).unwrap()["video_id"] + .as_str() + .unwrap() + .to_string(); + assert!(mock.confirmations().is_empty()); + + let app = sandbox.launch("recover", &[("CAP_GPUI_AUTO_RECOVER", "1")]); + wait_for_app( + &sandbox, + &app, + "the recovered upload to verify", + Duration::from_secs(240), + || state(&project).is_some_and(|state| state["phase"] == "Verified"), + ) + .await; + kill(app); + assert!(project.join("content/output.mp4").is_file()); + assert!( + mock.confirmations() + .iter() + .any(|confirmation| confirmation["videoId"] == video.as_str()) + ); + for index in 1..=4 { + assert!( + mock.object(&format!("{video}/segments/video/segment_{index:03}.m4s")) + .is_some(), + "segment {index} missing" + ); + } + assert!(!mock.deleted().contains(&video)); +} + +/// Stops the recording while its start is still in flight: the server video +/// it created is deleted and nothing is left behind. +#[ignore] +#[tokio::test(flavor = "multi_thread")] +async fn real_app_cancelled_start_deletes_its_video() { + let mock = MockCap::start(Vec::new()).await; + let sandbox = Sandbox::new(&mock.url, 2); + let app = sandbox.launch("cancel", &[("CAP_GPUI_AUTO_RECORD", "instant:2")]); + tokio::time::sleep(Duration::from_secs(20)).await; + kill(app); + let created = mock.created(); + assert_eq!(created.len(), 1, "{created:?}"); + if sandbox.log("cancel").contains("recording started") { + eprintln!("the stop landed after capture began; nothing to check"); + return; + } + assert_eq!(mock.deleted(), created); + assert!(sandbox.journal().is_empty(), "{:?}", sandbox.journal()); + assert!(sandbox.project_for(&created[0]).is_none()); +} From 10b281dcb622e420524baa62a68d22b49072cd46 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sat, 24 Oct 2026 16:12:35 +0100 Subject: [PATCH 3/7] ci: run the real-app Instant reliability suite on macOS --- .github/workflows/instant-reliability.yml | 153 ++++++++++++++++++++++ 1 file changed, 153 insertions(+) create mode 100644 .github/workflows/instant-reliability.yml diff --git a/.github/workflows/instant-reliability.yml b/.github/workflows/instant-reliability.yml new file mode 100644 index 0000000000..19e33b29c4 --- /dev/null +++ b/.github/workflows/instant-reliability.yml @@ -0,0 +1,153 @@ +name: Instant reliability (real app) +# Drives the real GPUI app through Instant recordings against the in-process +# mock Cap server (`apps/desktop-gpui/src/upload/real_app_tests.rs`): a normal +# recording, a start stopped while it is still in flight, force-kills from the +# server video's creation through the first seconds of capture, and a crash +# followed by Recover. Each case checks what reached the mock server and what +# was left on disk, so a regression that loses a recording, deletes one with +# media, or leaves an empty video behind fails here. + +on: + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + paths: + - "apps/desktop-gpui/src/upload.rs" + - "apps/desktop-gpui/src/upload/**" + - "apps/desktop-gpui/src/recording.rs" + - "apps/desktop-gpui/src/session.rs" + - "apps/desktop-gpui/src/library.rs" + - "apps/desktop-gpui/src/main_window.rs" + - "apps/desktop-gpui/src/auth.rs" + - "apps/desktop-gpui/src/store.rs" + - "crates/recording/src/upload_resume.rs" + - "crates/recording/src/upload_verification.rs" + - "crates/recording/src/instant_recording.rs" + - "crates/recording/src/recovery.rs" + - "crates/enc-ffmpeg/src/mux/segmented_stream.rs" + - "crates/enc-ffmpeg/src/mux/dash_audio.rs" + - ".github/workflows/instant-reliability.yml" + schedule: + - cron: "40 4 * * *" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: instant-reliability-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + real-app: + name: Instant crash, cancel and recovery (real app) + runs-on: blacksmith-6vcpu-macos-15 + timeout-minutes: 60 + env: + CARGO_PROFILE_DEV_DEBUG: "0" + TARGET: aarch64-apple-darwin + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Rust setup + uses: dtolnay/rust-toolchain@1.95.0 + with: + targets: aarch64-apple-darwin + + - uses: ./.github/actions/setup-rust-cache + with: + target: aarch64-apple-darwin + prefix-key: v1-rust-gpui + save-cache: "false" + workspaces: apps/desktop-gpui -> target + + - name: Run setup + run: node scripts/setup.js + env: + RUST_TARGET_TRIPLE: aarch64-apple-darwin + + - name: Prepare GPUI dependency + run: | + node scripts/sync-desktop-versions.mjs + ./scripts/prepare-gpui-dependency.sh + + - name: Build the app and its tests + run: | + set -euo pipefail + # The tests load frameworks from debug/Frameworks (beside deps/), the + # app binary from the directory above debug/. + for frameworks in \ + "$GITHUB_WORKSPACE/apps/desktop-gpui/target/$TARGET/debug/Frameworks" \ + "$GITHUB_WORKSPACE/apps/desktop-gpui/target/$TARGET/Frameworks"; do + mkdir -p "$frameworks" + ln -sfn "$GITHUB_WORKSPACE/target/Frameworks/Spacedrive.framework" "$frameworks/Spacedrive.framework" + done + cargo +1.95.0 build --manifest-path apps/desktop-gpui/Cargo.toml --locked --target "$TARGET" --bin cap-gpui + cargo +1.95.0 test --manifest-path apps/desktop-gpui/Cargo.toml --locked --target "$TARGET" --bin cap-gpui --no-run --message-format=json \ + | python3 -c 'import json, sys + for line in sys.stdin: + m = json.loads(line) + if m.get("reason") == "compiler-artifact" and m.get("executable") and m["target"]["name"] == "cap-gpui" and m["profile"]["test"]: + print(m["executable"])' > "$RUNNER_TEMP/test-binary" + test -s "$RUNNER_TEMP/test-binary" + echo "$GITHUB_WORKSPACE/apps/desktop-gpui/target/$TARGET/debug/cap-gpui" > "$RUNNER_TEMP/app-binary" + cat "$RUNNER_TEMP/test-binary" "$RUNNER_TEMP/app-binary" + + # Blacksmith runs a logged-in GUI session, but ScreenCaptureKit is denied + # until TCC grants the capturing binaries and the runner ancestry, the + # display starts asleep, and there is no audio device (the start chime + # plays before capture begins), as in verify-recording.yml. + - name: Capture permissions, display and audio device + run: | + set -euo pipefail + sudo pmset -a displaysleep 0 sleep 0 disksleep 0 + caffeinate -u -t 2 + nohup caffeinate -dimsu > /dev/null 2>&1 & + brew install --cask blackhole-2ch + brew install switchaudio-osx + sudo killall coreaudiod || true + for _ in $(seq 1 30); do + SwitchAudioSource -a -t output | grep -qx "BlackHole 2ch" && break + sleep 1 + done + SwitchAudioSource -s "BlackHole 2ch" -t output + SwitchAudioSource -s "BlackHole 2ch" -t input + cat "$RUNNER_TEMP/test-binary" "$RUNNER_TEMP/app-binary" > "$RUNNER_TEMP/capture-binaries" + p=$$ + while [ "$p" -gt 1 ]; do + ps -o comm= -p "$p" + p=$(ps -o ppid= -p "$p" | tr -d ' ') + done >> "$RUNNER_TEMP/capture-binaries" + db="/Library/Application Support/com.apple.TCC/TCC.db" + user_db="$HOME/Library/Application Support/com.apple.TCC/TCC.db" + approvals="$HOME/Library/Group Containers/group.com.apple.replayd/ScreenCaptureApprovals.plist" + mkdir -p "$(dirname "$approvals")" + now=$(date +%s) + until=$(date -u -v+1y '+%a %b %d %H:%M:%S GMT %Y') + sort -u "$RUNNER_TEMP/capture-binaries" | grep '^/' | while read -r client; do + for service in kTCCServiceScreenCapture kTCCServiceMicrophone; do + row="INSERT OR REPLACE INTO access (service,client,client_type,auth_value,auth_reason,auth_version,indirect_object_identifier,flags,last_modified) VALUES ('$service','$client',1,2,4,1,'UNUSED',0,$now);" + sudo sqlite3 "$db" "$row" + sqlite3 "$user_db" "$row" + done + /usr/libexec/PlistBuddy -c "Add :$client date $until" "$approvals" >/dev/null 2>&1 || true + echo "granted $client" + done + killall tccd || true + sudo killall tccd || true + sleep 2 + + - name: Real-app Instant reliability + run: | + set -euo pipefail + export CAP_GPUI_REAL_APP_BIN="$(cat "$RUNNER_TEMP/app-binary")" + export CAP_GPUI_REAL_APP_ARTIFACTS="$RUNNER_TEMP/real-app" + "$(cat "$RUNNER_TEMP/test-binary")" real_app --ignored --test-threads=1 --nocapture + + - name: Upload sandboxes + if: failure() + uses: actions/upload-artifact@v4 + with: + name: instant-reliability-sandboxes + path: ${{ runner.temp }}/real-app + retention-days: 7 From 6d5b890b974eda13ad136c2d5a73fef141f9c3bc Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sat, 24 Oct 2026 16:22:11 +0100 Subject: [PATCH 4/7] test(desktop-gpui): count pending segments as media, insist on a real cancelled start, and kill apps a failed case leaves running --- .../desktop-gpui/src/upload/real_app_tests.rs | 81 ++++++++++++++----- apps/desktop-gpui/src/upload/verify_tests.rs | 2 - 2 files changed, 61 insertions(+), 22 deletions(-) diff --git a/apps/desktop-gpui/src/upload/real_app_tests.rs b/apps/desktop-gpui/src/upload/real_app_tests.rs index 07734585ee..087786e990 100644 --- a/apps/desktop-gpui/src/upload/real_app_tests.rs +++ b/apps/desktop-gpui/src/upload/real_app_tests.rs @@ -66,7 +66,7 @@ impl Sandbox { Self { root } } - fn launch(&self, name: &str, env: &[(&str, &str)]) -> Child { + fn launch(&self, name: &str, env: &[(&str, &str)]) -> App { let binary = std::env::var("CAP_GPUI_REAL_APP_BIN") .expect("set CAP_GPUI_REAL_APP_BIN to the built cap-gpui binary"); let log = std::fs::File::create(self.root.path().join(format!("{name}.log"))).unwrap(); @@ -84,7 +84,7 @@ impl Sandbox { for (key, value) in env { command.env(key, value); } - command.spawn().unwrap() + App(Some(command.spawn().unwrap())) } fn log(&self, name: &str) -> String { @@ -172,10 +172,11 @@ fn media_files(project: &Path) -> usize { if depth > 0 { walk(&path, depth - 1) } else { 0 } } else { let sized = entry.metadata().is_ok_and(|metadata| metadata.len() > 0); + let segment = name.ends_with(".m4s") + || name.ends_with(".m4s.tmp") + || name.ends_with(".m4s.unsealed"); usize::from( - sized - && (name.ends_with(".m4s") - || (name.ends_with(".mp4") && !name.starts_with("init"))), + sized && (segment || (name.ends_with(".mp4") && !name.starts_with("init"))), ) } }) @@ -204,7 +205,7 @@ fn finished_segments(project: &Path) -> usize { /// hang can be diagnosed from the artifacts. async fn wait_for_app( sandbox: &Sandbox, - app: &Child, + app: &App, what: &str, timeout: Duration, mut done: impl FnMut() -> bool, @@ -228,9 +229,27 @@ async fn wait_for_app( } } -fn kill(mut app: Child) { - let _ = app.kill(); - let _ = app.wait(); +/// A launched app, killed when dropped, so a failing test never leaves it +/// running into the next one. +struct App(Option); + +impl App { + fn id(&self) -> u32 { + self.0.as_ref().map_or(0, Child::id) + } +} + +impl Drop for App { + fn drop(&mut self) { + if let Some(mut child) = self.0.take() { + let _ = child.kill(); + let _ = child.wait(); + } + } +} + +fn kill(app: App) { + drop(app); } /// A normal Instant recording uploads, verifies, and leaves nothing for the @@ -377,22 +396,44 @@ async fn real_app_recovered_recording_uploads() { } /// Stops the recording while its start is still in flight: the server video -/// it created is deleted and nothing is left behind. +/// it created is deleted and nothing is left behind. A stop that lands after +/// capture began exercises nothing, so the case retries until one lands +/// during the start, and fails if none does. #[ignore] #[tokio::test(flavor = "multi_thread")] async fn real_app_cancelled_start_deletes_its_video() { let mock = MockCap::start(Vec::new()).await; let sandbox = Sandbox::new(&mock.url, 2); - let app = sandbox.launch("cancel", &[("CAP_GPUI_AUTO_RECORD", "instant:2")]); - tokio::time::sleep(Duration::from_secs(20)).await; - kill(app); - let created = mock.created(); - assert_eq!(created.len(), 1, "{created:?}"); - if sandbox.log("cancel").contains("recording started") { - eprintln!("the stop landed after capture began; nothing to check"); + for attempt in 0..5 { + let name = format!("cancel-{attempt}"); + let before = mock.created().len(); + let app = sandbox.launch(&name, &[("CAP_GPUI_AUTO_RECORD", "instant:2")]); + wait_for_app( + &sandbox, + &app, + "the server video", + Duration::from_secs(60), + || mock.created().len() > before, + ) + .await; + tokio::time::sleep(Duration::from_secs(15)).await; + kill(app); + let video = mock.created()[before].clone(); + if sandbox.log(&name).contains("recording started") { + eprintln!("attempt {attempt}: the stop landed after capture began, retrying"); + continue; + } + assert!(mock.deleted().contains(&video), "{video} was not deleted"); + assert!(sandbox.project_for(&video).is_none()); + assert!( + sandbox + .journal() + .iter() + .all(|entry| entry["video_id"] != video.as_str()), + "{:?}", + sandbox.journal() + ); return; } - assert_eq!(mock.deleted(), created); - assert!(sandbox.journal().is_empty(), "{:?}", sandbox.journal()); - assert!(sandbox.project_for(&created[0]).is_none()); + panic!("no stop landed while the start was in flight"); } diff --git a/apps/desktop-gpui/src/upload/verify_tests.rs b/apps/desktop-gpui/src/upload/verify_tests.rs index 503978eeb2..23c67697be 100644 --- a/apps/desktop-gpui/src/upload/verify_tests.rs +++ b/apps/desktop-gpui/src/upload/verify_tests.rs @@ -435,12 +435,10 @@ impl MockCap { self.lock().confirmations.clone() } - /// Video ids the mock created, in order. pub(super) fn created(&self) -> Vec { self.lock().created.clone() } - /// Video ids the mock was asked to delete, in order. pub(super) fn deleted(&self) -> Vec { self.lock().deleted.clone() } From ae9cc36b79be4b60d849cd66054cea141d554802 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sat, 24 Oct 2026 16:47:14 +0100 Subject: [PATCH 5/7] test(desktop-gpui): note each crashed recording's state right after its kill and hold the sweep to it --- .../desktop-gpui/src/upload/real_app_tests.rs | 49 ++++++++++++++----- 1 file changed, 36 insertions(+), 13 deletions(-) diff --git a/apps/desktop-gpui/src/upload/real_app_tests.rs b/apps/desktop-gpui/src/upload/real_app_tests.rs index 087786e990..8db6686779 100644 --- a/apps/desktop-gpui/src/upload/real_app_tests.rs +++ b/apps/desktop-gpui/src/upload/real_app_tests.rs @@ -287,14 +287,23 @@ async fn real_app_instant_recording_uploads_and_verifies() { } /// Kills the app at points from the server video's creation through the -/// first seconds of capture, then relaunches long after. No video whose -/// recording holds media is ever deleted, and every crashed recording that -/// holds none and has its folder is deleted. +/// first seconds of capture, then relaunches long after. Each recording's state +/// is noted right after its kill, before anything else runs: a recording that +/// held media keeps its folder, its media and its server video, and every +/// crashed recording that held none in a folder still in progress has its +/// server video deleted. #[ignore] #[tokio::test(flavor = "multi_thread")] async fn real_app_crashes_never_delete_a_recording_with_media() { + struct Crashed { + video: String, + project: Option, + media: usize, + in_progress: bool, + } let mock = MockCap::start(Vec::new()).await; let sandbox = Sandbox::new(&mock.url, 2); + let mut crashed = Vec::new(); for (attempt, offset_ms) in [0u64, 150, 400, 800, 1500, 2500, 4000, 7000] .into_iter() .enumerate() @@ -314,6 +323,17 @@ async fn real_app_crashes_never_delete_a_recording_with_media() { .await; tokio::time::sleep(Duration::from_millis(offset_ms)).await; kill(app); + let video = mock.created()[created].clone(); + let project = sandbox.project_for(&video); + crashed.push(Crashed { + media: project.as_deref().map(media_files).unwrap_or(0), + in_progress: project + .as_deref() + .and_then(meta) + .is_some_and(|meta| meta.get("recording").is_some()), + video, + project, + }); } sandbox.age(); let app = sandbox.launch("sweep", &[]); @@ -321,19 +341,22 @@ async fn real_app_crashes_never_delete_a_recording_with_media() { kill(app); let deleted = mock.deleted(); - for video in mock.created() { - let project = sandbox.project_for(&video); - let media = project.as_deref().map(media_files).unwrap_or(0); - if media > 0 { + for recording in &crashed { + let video = &recording.video; + if recording.media > 0 { + assert!( + !deleted.contains(video), + "{video} was deleted although its recording held media" + ); + let project = recording.project.as_deref().unwrap(); + assert!(project.is_dir(), "{video}'s recording was removed"); assert!( - !deleted.contains(&video), - "{video} was deleted although its recording holds media" + media_files(project) >= recording.media, + "{video}'s recording lost media" ); - } else if let Some(project) = project - && meta(&project).is_some_and(|meta| meta.get("recording").is_some()) - { + } else if recording.project.is_some() && recording.in_progress { assert!( - deleted.contains(&video), + deleted.contains(video), "{video} crashed before capturing anything but was kept" ); } From 22e7f5b6efed66f951abdae9689188f8dd316929 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sat, 24 Oct 2026 16:47:14 +0100 Subject: [PATCH 6/7] ci: stop the Instant reliability job from running on edits to its own workflow alone --- .github/workflows/instant-reliability.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/instant-reliability.yml b/.github/workflows/instant-reliability.yml index 19e33b29c4..9153e6215a 100644 --- a/.github/workflows/instant-reliability.yml +++ b/.github/workflows/instant-reliability.yml @@ -25,7 +25,6 @@ on: - "crates/recording/src/recovery.rs" - "crates/enc-ffmpeg/src/mux/segmented_stream.rs" - "crates/enc-ffmpeg/src/mux/dash_audio.rs" - - ".github/workflows/instant-reliability.yml" schedule: - cron: "40 4 * * *" workflow_dispatch: From b6c35c5ed2fc082d2ac96e67d64c9d2ee0207b90 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sat, 24 Oct 2026 16:51:13 +0100 Subject: [PATCH 7/7] ci: run the Instant reliability job on its own edits from this repository, and on fork changes only when they touch Instant code --- .github/workflows/instant-reliability.yml | 35 +++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/.github/workflows/instant-reliability.yml b/.github/workflows/instant-reliability.yml index 9153e6215a..137864ea62 100644 --- a/.github/workflows/instant-reliability.yml +++ b/.github/workflows/instant-reliability.yml @@ -25,20 +25,55 @@ on: - "crates/recording/src/recovery.rs" - "crates/enc-ffmpeg/src/mux/segmented_stream.rs" - "crates/enc-ffmpeg/src/mux/dash_audio.rs" + - ".github/workflows/instant-reliability.yml" schedule: - cron: "40 4 * * *" workflow_dispatch: permissions: contents: read + pull-requests: read concurrency: group: instant-reliability-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true jobs: + # A pull request from a fork that changes only this workflow does not queue + # the macOS job; one that also changes Instant code does, and so does any + # change from this repository. + scope: + name: Scope + runs-on: blacksmith-2vcpu-ubuntu-2404 + timeout-minutes: 5 + outputs: + run: ${{ steps.scope.outputs.run }} + steps: + - name: Scope + id: scope + env: + GH_TOKEN: ${{ github.token }} + EVENT: ${{ github.event_name }} + PR: ${{ github.event.pull_request.number }} + SAME_REPO: ${{ github.event.pull_request.head.repo.full_name == github.repository }} + run: | + set -euo pipefail + if [[ "$EVENT" != pull_request || "$SAME_REPO" == true ]]; then + echo "run=true" >> "$GITHUB_OUTPUT"; exit 0 + fi + gh api "repos/$GITHUB_REPOSITORY/pulls/$PR/files" --paginate --jq '.[].filename' > "$RUNNER_TEMP/changed.txt" + instant='^(apps/desktop-gpui/src/(upload\.rs|upload/.*|recording\.rs|session\.rs|library\.rs|main_window\.rs|auth\.rs|store\.rs)|crates/recording/src/(upload_resume|upload_verification|instant_recording|recovery)\.rs|crates/enc-ffmpeg/src/mux/(segmented_stream|dash_audio)\.rs)$' + if grep -qE "$instant" "$RUNNER_TEMP/changed.txt"; then + echo "run=true" >> "$GITHUB_OUTPUT" + else + echo "run=false" >> "$GITHUB_OUTPUT" + echo "A fork pull request that changes no Instant code does not run the real-app suite." | tee -a "$GITHUB_STEP_SUMMARY" + fi + real-app: name: Instant crash, cancel and recovery (real app) + needs: scope + if: needs.scope.outputs.run == 'true' runs-on: blacksmith-6vcpu-macos-15 timeout-minutes: 60 env: