feat(nextjs): Wait for the resumed part of a document before starting the pageload

CHANGELOG.md

@@ -33,7 +33,7 @@
 
 - **feat(nextjs): Never emit trace meta tags from a prerender ([#25141](https://github.com/getsentry/sentry-javascript/pull/25141))**
 
-  Pageloads of apps with `cacheComponents` enabled are connected to their server request again. Pages that Next.js prerenders no longer carry trace information, so a cached page can no longer connect visitors to a request that was not theirs. This also applies to ISR pages.
+  Pageloads of apps with `cacheComponents` enabled are connected to their server request again. Pages that Next.js prerenders no longer carry trace information, so a cached page can no longer connect visitors to a request that was not theirs. This also applies to ISR pages. The pageload waits for the trace information when it arrives with the part of the page that streams in after a cached shell.
 
   If you set up OpenTelemetry yourself and register `SentryPropagator`, register `NextSentryPropagator` instead and pass your tracer provider to `markPrerendersOnTracerProvider`, so that your prerendered pages carry no trace information either.
 

dev-packages/e2e-tests/test-applications/nextjs-16-cacheComponents/app/blocking/page.tsx

@@ -0,0 +1,10 @@
+import { headers } from 'next/headers';
+
+// Opts out of the static shell: the whole page renders on the server for every request, so the
+// trace meta tags land in <head> like in a classic dynamic render.
+export const instant = false;
+
+export default async function Page() {
+  const requestHeaders = await headers();
+  return <h1 id="blocking">Blocking render for {requestHeaders.get('user-agent') ? 'a browser' : 'nobody'}</h1>;
+}

dev-packages/e2e-tests/test-applications/nextjs-16-cacheComponents/app/static-shell/page.tsx

@@ -0,0 +1,4 @@
+// No request data anywhere, so the whole page is a static shell and nothing renders per request.
+export default function Page() {
+  return <h1 id="static-shell">Static shell</h1>;
+}

dev-packages/e2e-tests/test-applications/nextjs-16-cacheComponents/tests/cacheOrigin-pageloadConnection.spec.ts

@@ -1,21 +1,24 @@
+import type { Page } from '@playwright/test';
 import { expect, test } from '@playwright/test';
 import { waitForTransaction } from '@sentry-internal/test-utils';
 
-test('connects the pageload trace to the runtime server request', async ({ page }) => {
-  // The resumed part of the document, which carries the tag, arrives after the SDK started. Enabled
-  // once the pageload instrumentation waits for it.
-  test.skip(true, 'Needs the deferred pageload start');
+const APP = 'nextjs-16-cacheComponents';
+
+// The server only writes `sentry-trace` meta tags for a render that happened for the current request,
+// never while it prerenders a shell. So a tag in the document can always be continued; the only
+// question is when it arrives. The resumed part of a prerendered shell arrives after the SDK started,
+// which the pageload instrumentation waits for.
 
+test('connects the pageload trace to the runtime server request', async ({ page }) => {
   // The shell is prerendered, but the page's dynamic part (`DynamicContent` awaits `headers()`)
   // runs on the server on every request and produces the `GET /pageload-tracing` transaction.
-  // A fully prerendered page would produce no server transaction.
-  const serverTxPromise = waitForTransaction('nextjs-16-cacheComponents', transactionEvent => {
+  const serverTxPromise = waitForTransaction(APP, transactionEvent => {
     return (
       transactionEvent.contexts?.trace?.op === 'http.server' && transactionEvent.transaction === 'GET /pageload-tracing'
     );
   });
 
-  const pageloadTxPromise = waitForTransaction('nextjs-16-cacheComponents', transactionEvent => {
+  const pageloadTxPromise = waitForTransaction(APP, transactionEvent => {
     return transactionEvent.contexts?.trace?.op === 'pageload' && transactionEvent.transaction === '/pageload-tracing';
   });
 
@@ -27,4 +30,83 @@ test('connects the pageload trace to the runtime server request', async ({ page
 
   expect(serverTx.contexts?.trace?.trace_id).toBeTruthy();
   expect(pageloadTx.contexts?.trace?.trace_id).toBe(serverTx.contexts?.trace?.trace_id);
+  expect(pageloadTx.contexts?.trace?.links).toBeUndefined();
+});
+
+test('connects the pageload trace to a blocking render', async ({ page }) => {
+  const serverTxPromise = waitForTransaction(APP, transactionEvent => {
+    return transactionEvent.contexts?.trace?.op === 'http.server' && transactionEvent.transaction === 'GET /blocking';
+  });
+
+  const pageloadTxPromise = waitForTransaction(APP, transactionEvent => {
+    return transactionEvent.contexts?.trace?.op === 'pageload' && transactionEvent.transaction === '/blocking';
+  });
+
+  await page.goto('/blocking');
+
+  await expect(page.locator('#blocking')).toHaveText('Blocking render for a browser');
+
+  const [serverTx, pageloadTx] = await Promise.all([serverTxPromise, pageloadTxPromise]);
+
+  // The whole page renders per request, so the tag in <head> is this request's and is continued.
+  expect(serverTx.contexts?.trace?.trace_id).toBeTruthy();
+  expect(pageloadTx.contexts?.trace?.trace_id).toBe(serverTx.contexts?.trace?.trace_id);
+  expect(pageloadTx.contexts?.trace?.links).toBeUndefined();
+});
+
+test('does not share a trace between visitors of a shell generated at request time', async ({ browser }) => {
+  test.skip(process.env.TEST_ENV !== 'production', 'Shells are only cached in production builds');
+
+  // `[id]` only lists `test` in `generateStaticParams`, so the first visit prerenders and caches the
+  // shell for this id. The prerender writes no trace tag, so every visitor starts a fresh trace.
+  const id = `shell-${Date.now()}`;
+
+  const visit = async (): Promise<{ traceId: string | undefined; links: unknown }> => {
+    const context = await browser.newContext();
+    const page: Page = await context.newPage();
+
+    const pageloadTxPromise = waitForTransaction(APP, transactionEvent => {
+      return (
+        transactionEvent.contexts?.trace?.op === 'pageload' &&
+        transactionEvent.contexts?.trace?.data?.['url.path'] === `/${id}`
+      );
+    });
+
+    await page.goto(`/${id}`);
+    await expect(page.locator('#result')).toHaveText(`Hello, ${id}!`);
+    expect(await page.locator('meta[name="sentry-trace"]').count()).toBe(0);
+
+    const pageloadTx = await pageloadTxPromise;
+    await context.close();
+
+    return { traceId: pageloadTx.contexts?.trace?.trace_id, links: pageloadTx.contexts?.trace?.links };
+  };
+
+  const first = await visit();
+  const second = await visit();
+
+  expect(first.traceId).toBeTruthy();
+  expect(second.traceId).toBeTruthy();
+  expect(second.traceId).not.toBe(first.traceId);
+  expect(first.links).toBeUndefined();
+  expect(second.links).toBeUndefined();
+});
+
+test('starts a fresh trace on a fully static page', async ({ page }) => {
+  test.skip(process.env.TEST_ENV !== 'production', 'Pages are only prerendered in production builds');
+
+  const pageloadTxPromise = waitForTransaction(APP, transactionEvent => {
+    return transactionEvent.contexts?.trace?.op === 'pageload' && transactionEvent.transaction === '/static-shell';
+  });
+
+  await page.goto('/static-shell');
+
+  await expect(page.locator('#static-shell')).toHaveText('Static shell');
+
+  const pageloadTx = await pageloadTxPromise;
+
+  // A build-time shell carries no trace context at all.
+  expect(await page.locator('meta[name="sentry-trace"]').count()).toBe(0);
+  expect(pageloadTx.contexts?.trace?.trace_id).toBeTruthy();
+  expect(pageloadTx.contexts?.trace?.links).toBeUndefined();
 });

dev-packages/e2e-tests/test-applications/nextjs-16-streaming-cacheComponents/app/blocking/page.tsx

@@ -0,0 +1,10 @@
+import { headers } from 'next/headers';
+
+// Opts out of the static shell: the whole page renders on the server for every request, so the
+// trace meta tags land in <head> like in a classic dynamic render.
+export const instant = false;
+
+export default async function Page() {
+  const requestHeaders = await headers();
+  return <h1 id="blocking">Blocking render for {requestHeaders.get('user-agent') ? 'a browser' : 'nobody'}</h1>;
+}

dev-packages/e2e-tests/test-applications/nextjs-16-streaming-cacheComponents/app/slow-hole/page.tsx

@@ -0,0 +1,24 @@
+import { headers } from 'next/headers';
+import Link from 'next/link';
+import { Suspense } from 'react';
+
+// The dynamic part, and with it the trace meta tag, streams in seconds after the shell, so a test
+// can navigate while the pageload still waits for the tag.
+async function SlowContent() {
+  await headers();
+  await new Promise(resolve => setTimeout(resolve, 4000));
+  return <p id="slow-content">Slow content</p>;
+}
+
+export default function Page() {
+  return (
+    <>
+      <Link id="navigate" href="/static-shell">
+        Static shell
+      </Link>
+      <Suspense fallback={<p id="slow-fallback">Loading...</p>}>
+        <SlowContent />
+      </Suspense>
+    </>
+  );
+}

dev-packages/e2e-tests/test-applications/nextjs-16-streaming-cacheComponents/app/static-shell/page.tsx

@@ -0,0 +1,4 @@
+// No request data anywhere, so the whole page is a static shell and nothing renders per request.
+export default function Page() {
+  return <h1 id="static-shell">Static shell</h1>;
+}

dev-packages/e2e-tests/test-applications/nextjs-16-streaming-cacheComponents/tests/cacheOrigin-pageloadConnection.spec.ts

@@ -1,19 +1,22 @@
+import type { Page } from '@playwright/test';
 import { expect, test } from '@playwright/test';
 import { getSpanOp, waitForStreamedSpan } from '@sentry-internal/test-utils';
 
-test('connects the pageload trace to the runtime server request', async ({ page }) => {
-  // The resumed part of the document, which carries the tag, arrives after the SDK started. Enabled
-  // once the pageload instrumentation waits for it.
-  test.skip(true, 'Needs the deferred pageload start');
+const APP = 'nextjs-16-streaming-cacheComponents';
+
+// The server only writes `sentry-trace` meta tags for a render that happened for the current request,
+// never while it prerenders a shell. So a tag in the document can always be continued; the only
+// question is when it arrives. The resumed part of a prerendered shell arrives after the SDK started,
+// which the pageload instrumentation waits for.
 
+test('connects the pageload trace to the runtime server request', async ({ page }) => {
   // The shell is prerendered, but the page's dynamic part (`DynamicContent` awaits `headers()`)
   // runs on the server on every request and produces the `GET /pageload-tracing` root span.
-  // A fully prerendered page would produce no server span.
-  const serverSpanPromise = waitForStreamedSpan('nextjs-16-streaming-cacheComponents', span => {
+  const serverSpanPromise = waitForStreamedSpan(APP, span => {
     return span.name === 'GET /pageload-tracing' && getSpanOp(span) === 'http.server' && span.is_segment;
   });
 
-  const pageloadSpanPromise = waitForStreamedSpan('nextjs-16-streaming-cacheComponents', span => {
+  const pageloadSpanPromise = waitForStreamedSpan(APP, span => {
     return span.name === '/pageload-tracing' && getSpanOp(span) === 'pageload' && span.is_segment;
   });
 
@@ -25,4 +28,111 @@ test('connects the pageload trace to the runtime server request', async ({ page
 
   expect(serverSpan.trace_id).toBeTruthy();
   expect(pageloadSpan.trace_id).toBe(serverSpan.trace_id);
+  expect(pageloadSpan.links).toBeUndefined();
+});
+
+test('connects the pageload trace to a blocking render', async ({ page }) => {
+  const serverSpanPromise = waitForStreamedSpan(APP, span => {
+    return span.name === 'GET /blocking' && getSpanOp(span) === 'http.server' && span.is_segment;
+  });
+
+  const pageloadSpanPromise = waitForStreamedSpan(APP, span => {
+    return span.name === '/blocking' && getSpanOp(span) === 'pageload' && span.is_segment;
+  });
+
+  await page.goto('/blocking');
+
+  await expect(page.locator('#blocking')).toHaveText('Blocking render for a browser');
+
+  const [serverSpan, pageloadSpan] = await Promise.all([serverSpanPromise, pageloadSpanPromise]);
+
+  // The whole page renders per request, so the tag in <head> is this request's and is continued.
+  expect(serverSpan.trace_id).toBeTruthy();
+  expect(pageloadSpan.trace_id).toBe(serverSpan.trace_id);
+  expect(pageloadSpan.links).toBeUndefined();
+});
+
+test('does not share a trace between visitors of a shell generated at request time', async ({ browser }) => {
+  test.skip(process.env.TEST_ENV !== 'production', 'Shells are only cached in production builds');
+
+  // `[id]` only lists `test` in `generateStaticParams`, so the first visit prerenders and caches the
+  // shell for this id. The prerender writes no trace tag, so every visitor starts a fresh trace.
+  const id = `shell-${Date.now()}`;
+
+  const visit = async (): Promise<{ traceId: string | undefined; links: unknown }> => {
+    const context = await browser.newContext();
+    const page: Page = await context.newPage();
+
+    const pageloadSpanPromise = waitForStreamedSpan(APP, span => {
+      return getSpanOp(span) === 'pageload' && span.is_segment && span.attributes['url.path']?.value === `/${id}`;
+    });
+
+    await page.goto(`/${id}`);
+    await expect(page.locator('#result')).toHaveText(`Hello, ${id}!`);
+    expect(await page.locator('meta[name="sentry-trace"]').count()).toBe(0);
+
+    const pageloadSpan = await pageloadSpanPromise;
+    await context.close();
+
+    return { traceId: pageloadSpan.trace_id, links: pageloadSpan.links };
+  };
+
+  const first = await visit();
+  const second = await visit();
+
+  expect(first.traceId).toBeTruthy();
+  expect(second.traceId).toBeTruthy();
+  expect(second.traceId).not.toBe(first.traceId);
+  expect(first.links).toBeUndefined();
+  expect(second.links).toBeUndefined();
+});
+
+test('starts a fresh trace on a fully static page', async ({ page }) => {
+  test.skip(process.env.TEST_ENV !== 'production', 'Pages are only prerendered in production builds');
+
+  const pageloadSpanPromise = waitForStreamedSpan(APP, span => {
+    return span.name === '/static-shell' && getSpanOp(span) === 'pageload' && span.is_segment;
+  });
+
+  await page.goto('/static-shell');
+
+  await expect(page.locator('#static-shell')).toHaveText('Static shell');
+
+  const pageloadSpan = await pageloadSpanPromise;
+
+  // A build-time shell carries no trace context at all.
+  expect(await page.locator('meta[name="sentry-trace"]').count()).toBe(0);
+  expect(pageloadSpan.trace_id).toBeTruthy();
+  expect(pageloadSpan.links).toBeUndefined();
+});
+
+test('keeps a navigation that starts while the pageload waits for the trace meta tag', async ({ page }) => {
+  test.skip(process.env.TEST_ENV !== 'production', 'Only a prerendered shell makes the trace meta tag arrive late');
+
+  const pageloadSpanPromise = waitForStreamedSpan(APP, span => {
+    return span.name === '/slow-hole' && getSpanOp(span) === 'pageload' && span.is_segment;
+  });
+
+  const navigationSpanPromise = waitForStreamedSpan(APP, span => {
+    return span.name === '/static-shell' && getSpanOp(span) === 'navigation' && span.is_segment;
+  });
+
+  await page.goto('/slow-hole', { waitUntil: 'commit' });
+  await page.waitForFunction(() => !!(window as { next?: { router?: unknown } }).next?.router);
+
+  // The resumed part, which carries the tag, has not arrived yet, so the pageload is still waiting.
+  await expect(page.locator('#slow-fallback')).toBeVisible();
+  expect(await page.locator('meta[name="sentry-trace"]').count()).toBe(0);
+
+  await page.locator('#navigate').click();
+  await expect(page.locator('#static-shell')).toHaveText('Static shell');
+
+  const [pageloadSpan, navigationSpan] = await Promise.all([pageloadSpanPromise, navigationSpanPromise]);
+
+  // The navigation ends the pageload like any pageload it interrupts, and runs to its own idle timeout.
+  expect(pageloadSpan.attributes['sentry.idle_span_finish_reason']).toEqual({ type: 'string', value: 'cancelled' });
+  expect(navigationSpan.attributes['sentry.idle_span_finish_reason']).toEqual({
+    type: 'string',
+    value: 'idleTimeout',
+  });
 });

packages/nextjs/src/client/browserTracingIntegration.ts

@@ -1,5 +1,6 @@
 import type { Integration } from '@sentry/core';
 import { browserTracingIntegration as originalBrowserTracingIntegration, isBotUserAgent } from '@sentry/react';
+import { settlePendingPageloadWait } from './routing/appRouterRoutingInstrumentation';
 import { nextRouterInstrumentNavigation, nextRouterInstrumentPageLoad } from './routing/nextRoutingInstrumentation';
 
 /**
@@ -41,6 +42,13 @@ export function browserTracingIntegration(
         nextRouterInstrumentNavigation(client);
       }
 
+      // A pageload that is still waiting for its trace meta tag has to exist before the navigation span,
+      // so the navigation ends it like any pageload it interrupts. Started later, the pageload would end
+      // the navigation span instead.
+      if (instrumentPageLoad) {
+        client.on('beforeStartNavigationSpan', settlePendingPageloadWait);
+      }
+
       browserTracingIntegrationInstance.afterAllSetup(client);
 
       if (instrumentPageLoad) {