fix(bridge): fail a missing image snapshot in CI mode instead of writing it

.gitignore

@@ -112,6 +112,8 @@ yarn-error.log
 # testing
 /coverage
 .harness/
+# Images a failed match saves next to the playground's committed image snapshots
+/apps/playground/**/__image_snapshots__/**/__diff_output__/
 # Prebuilt XCTest agent artifacts, packed by the platform-apple `prepack` script
 /packages/platform-ios/xctest-agent-prebuilds/
 vite.config.*.timestamp*

.nx/version-plans/version-plan-1792417138768.md

@@ -0,0 +1,5 @@
+---
+__default__: minor
+---
+
+In Jest's CI mode (on by default in CI), `toMatchImageSnapshot` fails when its snapshot does not exist instead of writing it and passing, and saves the received image beside it as `<name>-actual.png`. A project whose CI relied on missing snapshots being written commits those images as its snapshots, or runs with `--ci=false` on a machine with the comparing device. Like Jest's own snapshot matchers, `toMatchImageSnapshot` cannot be used with `.not`.

packages/bridge/src/__tests__/image-snapshot.test.ts

@@ -1,10 +1,166 @@
-import { afterEach, beforeEach, describe, expect, it } from 'vitest';
-import fs from 'node:fs/promises';
+import fs, { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
+import { existsSync } from 'node:fs';
 import os from 'node:os';
 import path from 'node:path';
 import { PNG } from 'pngjs';
+import { afterEach, beforeEach, describe, expect, it } from 'vitest';
 import { matchImageSnapshot } from '../image-snapshot.js';
 
+const createPng = ({ width = 2, height = 2, fill = 255 } = {}): Buffer => {
+  const png = new PNG({ width, height });
+  png.data.fill(fill);
+  return PNG.sync.write(png);
+};
+
+describe('matchImageSnapshot', () => {
+  let directory: string;
+  let testFilePath: string;
+  let receivedPath: string;
+  let snapshotPath: string;
+
+  beforeEach(async () => {
+    directory = await mkdtemp(
+      path.join(os.tmpdir(), 'harness-image-snapshot-')
+    );
+    testFilePath = path.join(directory, 'screen.harness.tsx');
+    receivedPath = path.join(directory, 'received.png');
+    snapshotPath = path.join(
+      directory,
+      '__image_snapshots__',
+      'android',
+      'square.png'
+    );
+    await writeFile(receivedPath, createPng());
+  });
+
+  afterEach(async () => {
+    await rm(directory, { force: true, recursive: true });
+  });
+
+  const diffOutputPath = (file: string): string =>
+    path.join(path.dirname(snapshotPath), '__diff_output__', file);
+
+  it('writes a missing snapshot and passes when new snapshots may be written', async () => {
+    const result = await matchImageSnapshot(
+      { path: receivedPath },
+      testFilePath,
+      { name: 'square' },
+      'android',
+      'new'
+    );
+
+    expect(result.pass).toBe(true);
+    expect(await readFile(snapshotPath)).toEqual(await readFile(receivedPath));
+    expect(existsSync(diffOutputPath('square-actual.png'))).toBe(false);
+  });
+
+  it('writes a missing snapshot and passes when snapshots are updated', async () => {
+    const result = await matchImageSnapshot(
+      { path: receivedPath },
+      testFilePath,
+      { name: 'square' },
+      'android',
+      'all'
+    );
+
+    expect(result.pass).toBe(true);
+    expect(await readFile(snapshotPath)).toEqual(await readFile(receivedPath));
+    expect(existsSync(diffOutputPath('square-actual.png'))).toBe(false);
+  });
+
+  it('fails a missing snapshot without writing it in CI mode, and saves the received image to __diff_output__', async () => {
+    const actualPath = diffOutputPath('square-actual.png');
+
+    const result = await matchImageSnapshot(
+      { path: receivedPath },
+      testFilePath,
+      { name: 'square' },
+      'android',
+      'none'
+    );
+
+    expect(result.pass).toBe(false);
+    expect(result.message).toContain(
+      `New snapshot was not written: ${snapshotPath} does not exist`
+    );
+    expect(result.message).toContain(`Received image saved at ${actualPath}`);
+    expect(existsSync(snapshotPath)).toBe(false);
+    expect(await readFile(actualPath)).toEqual(await readFile(receivedPath));
+  });
+
+  it('compares an existing snapshot in CI mode', async () => {
+    await matchImageSnapshot(
+      { path: receivedPath },
+      testFilePath,
+      { name: 'square' },
+      'android',
+      'new'
+    );
+
+    const result = await matchImageSnapshot(
+      { path: receivedPath },
+      testFilePath,
+      { name: 'square' },
+      'android',
+      'none'
+    );
+
+    expect(result).toEqual({ pass: true, message: 'Images match' });
+    expect(existsSync(diffOutputPath('square-actual.png'))).toBe(false);
+    expect(existsSync(diffOutputPath('square-diff.png'))).toBe(false);
+  });
+
+  it('saves the diff and received images of a mismatch to __diff_output__', async () => {
+    await mkdir(path.dirname(snapshotPath), { recursive: true });
+    await writeFile(snapshotPath, createPng({ fill: 0 }));
+
+    const result = await matchImageSnapshot(
+      { path: receivedPath },
+      testFilePath,
+      { name: 'square' },
+      'android',
+      'none'
+    );
+
+    expect(result.pass).toBe(false);
+    expect(await readFile(diffOutputPath('square-actual.png'))).toEqual(
+      await readFile(receivedPath)
+    );
+    expect(existsSync(diffOutputPath('square-diff.png'))).toBe(true);
+  });
+
+  it('names the received image of a missing snapshot after the whole snapshot name', async () => {
+    await matchImageSnapshot(
+      { path: receivedPath },
+      testFilePath,
+      { name: 'logo.png-dark' },
+      'android',
+      'none'
+    );
+
+    expect(existsSync(diffOutputPath('logo.png-dark-actual.png'))).toBe(true);
+  });
+
+  it('names the images of a mismatch after the whole snapshot name', async () => {
+    await mkdir(path.dirname(snapshotPath), { recursive: true });
+    await writeFile(
+      path.join(path.dirname(snapshotPath), 'logo.png-dark.png'),
+      createPng({ fill: 0 })
+    );
+
+    await matchImageSnapshot(
+      { path: receivedPath },
+      testFilePath,
+      { name: 'logo.png-dark' },
+      'android',
+      'none'
+    );
+
+    expect(existsSync(diffOutputPath('logo.png-dark-actual.png'))).toBe(true);
+    expect(existsSync(diffOutputPath('logo.png-dark-diff.png'))).toBe(true);
+  });
+});
+
 const solid = (r: number, g: number, b: number) => {
   const png = new PNG({ width: 4, height: 4 });
   for (let i = 0; i < png.data.length; i += 4) {
@@ -52,7 +208,8 @@ describe('matchImageSnapshot failure output', () => {
       { path: received },
       testFile,
       { name: 'code' },
-      'ios'
+      'ios',
+      'new'
     );
 
     const outDir = path.join(snapshotsDir, '__diff_output__');
@@ -90,7 +247,8 @@ describe('matchImageSnapshot failure output', () => {
       { path: received },
       testFile,
       { name: 'box' },
-      'ios'
+      'ios',
+      'new'
     );
 
     const actualPath = path.join(

packages/bridge/src/image-snapshot.ts

@@ -3,7 +3,11 @@ import { ssim } from 'ssim.js';
 import fs from 'node:fs/promises';
 import path from 'node:path';
 import { PNG } from 'pngjs';
-import type { FileReference, ImageSnapshotOptions } from './shared.js';
+import type {
+  FileReference,
+  ImageSnapshotOptions,
+  SnapshotUpdateState,
+} from './shared.js';
 
 type PixelmatchOptions = Parameters<typeof pixelmatch>[5];
 
@@ -50,7 +54,8 @@ export const matchImageSnapshot = async (
   screenshot: FileReference,
   testFilePath: string,
   options: ImageSnapshotOptions,
-  platformName: string
+  platformName: string,
+  updateSnapshot: SnapshotUpdateState
 ) => {
   const pixelmatchOptions = {
     ...DEFAULT_OPTIONS_FOR_PIXELMATCH,
@@ -70,14 +75,28 @@ export const matchImageSnapshot = async (
   const testDir = path.dirname(testFilePath);
   const snapshotsDir = path.join(testDir, SNAPSHOT_DIR_NAME, platformName);
 
-  const snapshotName = `${options.name}.png`;
-  const snapshotPath = path.join(snapshotsDir, snapshotName);
+  const snapshotPath = path.join(snapshotsDir, `${options.name}.png`);
+  // Failure artifacts live in their own folder so they can never overwrite
+  // a baseline of another snapshot (e.g. one named `<name>-diff`).
+  const diffOutputDir = path.join(snapshotsDir, DIFF_OUTPUT_DIR_NAME);
+  const failureImagePath = (image: 'actual' | 'diff'): string =>
+    path.join(diffOutputDir, `${options.name}-${image}.png`);
+  const actualPath = failureImagePath('actual');
 
   await fs.mkdir(snapshotsDir, { recursive: true });
 
   try {
     await fs.access(snapshotPath);
   } catch {
+    if (updateSnapshot === 'none') {
+      await fs.mkdir(diffOutputDir, { recursive: true });
+      await fs.writeFile(actualPath, receivedBuffer);
+      return {
+        pass: false,
+        message: `New snapshot was not written: ${snapshotPath} does not exist, and missing snapshots are not written in CI mode. Received image saved at ${actualPath}; commit it as ${snapshotPath}, or run without CI mode (--ci=false) on a machine with this device and commit the snapshot it writes.`,
+      };
+    }
+
     // First time - create snapshot
     await fs.writeFile(snapshotPath, receivedBuffer);
     return {
@@ -94,13 +113,6 @@ export const matchImageSnapshot = async (
   const img2 = PNG.sync.read(snapshotBuffer);
   const { width, height } = img1;
   const diff = new PNG({ width, height });
-  // Failure artifacts live in their own folder so they can never overwrite
-  // a baseline of another snapshot (e.g. one named `<name>-diff`).
-  const diffOutputDir = path.join(snapshotsDir, DIFF_OUTPUT_DIR_NAME);
-  const actualPath = path.join(
-    diffOutputDir,
-    `${snapshotName.replace('.png', '')}-actual.png`
-  );
 
   if (img1.width !== img2.width || img1.height !== img2.height) {
     await fs.mkdir(diffOutputDir, { recursive: true });
@@ -165,8 +177,7 @@ export const matchImageSnapshot = async (
   if (!pass) {
     await fs.mkdir(diffOutputDir, { recursive: true });
 
-    const diffFileName = `${snapshotName.replace('.png', '')}-diff.png`;
-    const diffPath = path.join(diffOutputDir, diffFileName);
+    const diffPath = failureImagePath('diff');
     await fs.writeFile(diffPath, PNG.sync.write(diff));
 
     await fs.writeFile(actualPath, receivedBuffer);

packages/bridge/src/server.ts

@@ -180,6 +180,7 @@ export const createHarnessBridge = async (
               testPath,
               opts,
               context.platform.name,
+              context.updateSnapshot ?? 'new'
             );
           } finally {
             // Only delete files this bridge created, never a client-supplied path.

packages/bridge/src/shared.ts

@@ -169,6 +169,10 @@ export type BridgeServerFunctions = {
   ) => Promise<{ pass: boolean; message: string }>;
 };
 
+export type SnapshotUpdateState = 'all' | 'new' | 'none';
+
 export type HarnessContext = {
   platform: HarnessPlatform;
+  /** Jest's snapshot update state; when absent, a missing image snapshot is written. */
+  updateSnapshot?: SnapshotUpdateState;
 };

packages/cli/skills/ui.md

@@ -37,7 +37,8 @@ Use this skill when the task requires:
 
 - Keep imports split correctly: core APIs from `react-native-harness`, UI APIs from `@react-native-harness/ui`.
 - Mention that `@react-native-harness/ui` requires installation, and native apps must be rebuilt after adding it.
-- `toMatchImageSnapshot(...)` needs a unique snapshot `name`.
+- `toMatchImageSnapshot(...)` needs a unique snapshot `name`, and cannot be used with `.not`.
+- In Jest's CI mode, on by default in CI, a missing snapshot fails and its received image is saved as `<name>-actual.png`: commit that image as `<name>.png`, or run with `--ci=false` on the device that compares it and commit the snapshot it writes.
 - If screenshotting elements that extend beyond screen bounds, call out `disableViewFlattening: true` in `rn-harness.config.mjs`.
 - On web, UI interactions and screenshots run through the web runner's Playwright-backed browser environment.