From b306c163bc91073dc9af5e64966c4c0bf8e480f6 Mon Sep 17 00:00:00 2001 From: Zhangyi Yuan Date: Thu, 8 Oct 2026 15:19:14 +0800 Subject: [PATCH 1/3] fix: fail TypeScript docs validation on compiler errors --- .github/workflows/sdk-nodejs.yml | 1 + scripts/docs-validation/package.json | 1 + scripts/docs-validation/validate.test.mjs | 87 +++++++++++++++++++++++ scripts/docs-validation/validate.ts | 14 ++-- 4 files changed, 98 insertions(+), 5 deletions(-) create mode 100644 scripts/docs-validation/validate.test.mjs diff --git a/.github/workflows/sdk-nodejs.yml b/.github/workflows/sdk-nodejs.yml index 9fbbf97e76..9218c9b3f7 100644 --- a/.github/workflows/sdk-nodejs.yml +++ b/.github/workflows/sdk-nodejs.yml @@ -119,6 +119,7 @@ jobs: working-directory: ${{ inputs.sdk-home }}/scripts/docs-validation run: | npm ci + npm test npm run extract npm run validate:ts diff --git a/scripts/docs-validation/package.json b/scripts/docs-validation/package.json index a7e881ba4f..82f0312d84 100644 --- a/scripts/docs-validation/package.json +++ b/scripts/docs-validation/package.json @@ -4,6 +4,7 @@ "private": true, "type": "module", "scripts": { + "test": "node --test validate.test.mjs", "extract": "tsx extract.ts", "validate": "tsx validate.ts", "validate:ts": "tsx validate.ts --lang=typescript", diff --git a/scripts/docs-validation/validate.test.mjs b/scripts/docs-validation/validate.test.mjs new file mode 100644 index 0000000000..6fc4585025 --- /dev/null +++ b/scripts/docs-validation/validate.test.mjs @@ -0,0 +1,87 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import { createRequire } from "node:module"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; +import { fileURLToPath, pathToFileURL } from "node:url"; + +const directory = path.dirname(fileURLToPath(import.meta.url)); +const require = createRequire(import.meta.url); +const compilerDirectory = path.dirname(require.resolve("typescript/package.json")); +const loader = pathToFileURL(require.resolve("tsx")).href; + +function validate(context, { source = "export const value: number = 42;", types = true, compiler = true, external = false } = {}) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "sdk docs validation ")); + context.after(() => fs.rmSync(root, { recursive: true, force: true })); + const tooling = path.join(root, "scripts/docs-validation"); + const validation = path.join(root, "docs/.validation"); + const examples = path.join(validation, "typescript"); + fs.mkdirSync(tooling, { recursive: true }); + fs.mkdirSync(examples, { recursive: true }); + fs.mkdirSync(path.join(root, "nodejs/node_modules"), { recursive: true }); + const linkType = process.platform === "win32" ? "junction" : "dir"; + fs.symlinkSync(path.join(directory, "node_modules"), path.join(tooling, "node_modules"), linkType); + if (compiler) { + fs.symlinkSync(compilerDirectory, path.join(root, "nodejs/node_modules/typescript"), linkType); + } + if (types) { + const typeDirectory = path.join(root, "node_modules/@types/node"); + fs.mkdirSync(typeDirectory, { recursive: true }); + fs.writeFileSync(path.join(typeDirectory, "index.d.ts"), "export {};\n"); + } + fs.copyFileSync(path.join(directory, "validate.ts"), path.join(tooling, "validate.ts")); + fs.writeFileSync(path.join(root, "package.json"), JSON.stringify({ type: "module" })); + fs.writeFileSync(path.join(examples, "sample.ts"), source); + if (external) { + fs.writeFileSync(path.join(root, "broken.ts"), 'export const value: number = "invalid";'); + } + fs.writeFileSync(path.join(validation, "manifest.json"), JSON.stringify({ blocks: [{ + sourceFile: "fixture.md", sourceLine: 7, outputFile: "typescript/sample.ts", + }] })); + const environment = { ...process.env }; + delete environment.GITHUB_STEP_SUMMARY; + const result = spawnSync(process.execPath, [ + "--import", loader, path.join(tooling, "validate.ts"), "--lang=typescript", + ], { cwd: root, encoding: "utf8", env: environment, timeout: 60000 }); + assert.ifError(result.error); + assert.equal(result.signal, null); + return { status: result.status, output: result.stdout + result.stderr }; +} + +test("valid TypeScript succeeds in a path containing spaces", (context) => { + const result = validate(context); + assert.equal(result.status, 0, result.output); + assert.match(result.output, /1 files passed/); +}); + +test("example type errors fail and retain their documentation location", (context) => { + const result = validate(context, { source: 'export const value: number = "invalid";' }); + assert.equal(result.status, 1, result.output); + assert.match(result.output, /TS2322/); + assert.match(result.output, /fixture\.md:7/); +}); + +test("global compiler diagnostics fail validation", (context) => { + const result = validate(context, { types: false }); + assert.equal(result.status, 1, result.output); + assert.match(result.output, /TS2688/); + assert.doesNotMatch(result.output, /All documentation code blocks are valid/); +}); + +test("a missing compiler fails validation", (context) => { + const result = validate(context, { compiler: false }); + assert.equal(result.status, 1, result.output); + assert.match(result.output, /Cannot find module|MODULE_NOT_FOUND/); + assert.doesNotMatch(result.output, /All documentation code blocks are valid/); +}); + +test("errors outside extracted examples fail validation", (context) => { + const result = validate(context, { + source: 'export { value } from "../../../broken.js";', external: true, + }); + assert.equal(result.status, 1, result.output); + assert.match(result.output, /broken\.ts/); + assert.match(result.output, /TS2322/); +}); \ No newline at end of file diff --git a/scripts/docs-validation/validate.ts b/scripts/docs-validation/validate.ts index cfa14b9251..25b66750b3 100644 --- a/scripts/docs-validation/validate.ts +++ b/scripts/docs-validation/validate.ts @@ -75,8 +75,8 @@ async function validateTypeScript(): Promise { try { // Run tsc - const tscPath = path.join(ROOT_DIR, "nodejs/node_modules/.bin/tsc"); - execFileSync(tscPath, ["--project", tsconfigPath], { + const tscPath = path.join(ROOT_DIR, "nodejs/node_modules/typescript/bin/tsc"); + execFileSync(process.execPath, [tscPath, "--project", tsconfigPath, "--pretty", "false"], { encoding: "utf-8", cwd: tsDir, }); @@ -98,7 +98,7 @@ async function validateTypeScript(): Promise { } } catch (err: any) { // Parse tsc output for errors - const output = err.stdout || err.stderr || err.message || ""; + const output = [err.stdout, err.stderr].filter(Boolean).join("\n") || err.message || ""; const errorLines = output.split("\n"); const fileErrors = new Map(); let currentFile = ""; @@ -106,7 +106,7 @@ async function validateTypeScript(): Promise { for (const line of errorLines) { const match = line.match(/^(.+\.ts)\((\d+),(\d+)\): error/); if (match) { - currentFile = match[1]; + currentFile = path.resolve(tsDir, match[1]); if (!fileErrors.has(currentFile)) { fileErrors.set(currentFile, []); } @@ -124,7 +124,7 @@ async function validateTypeScript(): Promise { const block = manifest.blocks.find( (b) => b.outputFile === `typescript/${file}`, ); - const errors = fileErrors.get(fullPath) || fileErrors.get(file) || []; + const errors = fileErrors.get(fullPath) || []; results.push({ file: `typescript/${file}`, @@ -134,6 +134,10 @@ async function validateTypeScript(): Promise { errors, }); } + + if (results.every((result) => result.success)) { + throw new Error(`TypeScript documentation compilation failed:\n${output}`); + } } return results; From 2273a4b139da7eba8d83b781198dbf3a95343755 Mon Sep 17 00:00:00 2001 From: Zhangyi Yuan Date: Thu, 8 Oct 2026 16:24:32 +0800 Subject: [PATCH 2/3] fix: quote Go documentation validation module paths --- .github/workflows/sdk-go.yml | 1 + scripts/docs-validation/validate.test.mjs | 37 ++++++++++++++++++----- scripts/docs-validation/validate.ts | 2 +- 3 files changed, 32 insertions(+), 8 deletions(-) diff --git a/.github/workflows/sdk-go.yml b/.github/workflows/sdk-go.yml index e7235998f8..3e39743fe5 100644 --- a/.github/workflows/sdk-go.yml +++ b/.github/workflows/sdk-go.yml @@ -95,6 +95,7 @@ jobs: working-directory: ${{ inputs.sdk-home }}/scripts/docs-validation run: | npm ci + npm test npm run extract npm run validate:go diff --git a/scripts/docs-validation/validate.test.mjs b/scripts/docs-validation/validate.test.mjs index 6fc4585025..8b2859cd85 100644 --- a/scripts/docs-validation/validate.test.mjs +++ b/scripts/docs-validation/validate.test.mjs @@ -12,12 +12,12 @@ const require = createRequire(import.meta.url); const compilerDirectory = path.dirname(require.resolve("typescript/package.json")); const loader = pathToFileURL(require.resolve("tsx")).href; -function validate(context, { source = "export const value: number = 42;", types = true, compiler = true, external = false } = {}) { +function validate(context, { source = "export const value: number = 42;", types = true, compiler = true, external = false, language = "typescript" } = {}) { const root = fs.mkdtempSync(path.join(os.tmpdir(), "sdk docs validation ")); context.after(() => fs.rmSync(root, { recursive: true, force: true })); const tooling = path.join(root, "scripts/docs-validation"); const validation = path.join(root, "docs/.validation"); - const examples = path.join(validation, "typescript"); + const examples = path.join(validation, language); fs.mkdirSync(tooling, { recursive: true }); fs.mkdirSync(examples, { recursive: true }); fs.mkdirSync(path.join(root, "nodejs/node_modules"), { recursive: true }); @@ -33,21 +33,23 @@ function validate(context, { source = "export const value: number = 42;", types } fs.copyFileSync(path.join(directory, "validate.ts"), path.join(tooling, "validate.ts")); fs.writeFileSync(path.join(root, "package.json"), JSON.stringify({ type: "module" })); - fs.writeFileSync(path.join(examples, "sample.ts"), source); + if (source !== null) { + fs.writeFileSync(path.join(examples, "sample.ts"), source); + } if (external) { fs.writeFileSync(path.join(root, "broken.ts"), 'export const value: number = "invalid";'); } - fs.writeFileSync(path.join(validation, "manifest.json"), JSON.stringify({ blocks: [{ + fs.writeFileSync(path.join(validation, "manifest.json"), JSON.stringify({ blocks: source === null ? [] : [{ sourceFile: "fixture.md", sourceLine: 7, outputFile: "typescript/sample.ts", }] })); const environment = { ...process.env }; delete environment.GITHUB_STEP_SUMMARY; const result = spawnSync(process.execPath, [ - "--import", loader, path.join(tooling, "validate.ts"), "--lang=typescript", + "--import", loader, path.join(tooling, "validate.ts"), `--lang=${language}`, ], { cwd: root, encoding: "utf8", env: environment, timeout: 60000 }); assert.ifError(result.error); assert.equal(result.signal, null); - return { status: result.status, output: result.stdout + result.stderr }; + return { status: result.status, output: result.stdout + result.stderr, root }; } test("valid TypeScript succeeds in a path containing spaces", (context) => { @@ -84,4 +86,25 @@ test("errors outside extracted examples fail validation", (context) => { assert.equal(result.status, 1, result.output); assert.match(result.output, /broken\.ts/); assert.match(result.output, /TS2322/); -}); \ No newline at end of file +}); + +test("generated Go module preserves a checkout path containing spaces", (context) => { + const version = spawnSync("go", ["version"], { encoding: "utf8" }); + if (version.error?.code === "ENOENT") { + context.skip("Go is not installed"); + return; + } + assert.ifError(version.error); + assert.equal(version.status, 0, version.stderr); + const result = validate(context, { language: "go", source: null }); + assert.equal(result.status, 0, result.output); + const parsed = spawnSync("go", ["mod", "edit", "-json"], { + cwd: path.join(result.root, "docs/.validation/go"), encoding: "utf8", + }); + assert.ifError(parsed.error); + assert.equal(parsed.status, 0, parsed.stderr); + const replacement = JSON.parse(parsed.stdout).Replace.find( + (entry) => entry.Old.Path === "github.com/github/copilot-sdk/go", + ); + assert.equal(replacement.New.Path, path.join(result.root, "go")); +}); diff --git a/scripts/docs-validation/validate.ts b/scripts/docs-validation/validate.ts index 25b66750b3..8e82e43b28 100644 --- a/scripts/docs-validation/validate.ts +++ b/scripts/docs-validation/validate.ts @@ -230,7 +230,7 @@ go 1.21 require github.com/github/copilot-sdk/go v0.0.0 -replace github.com/github/copilot-sdk/go => ${path.join(ROOT_DIR, "go")} +replace github.com/github/copilot-sdk/go => ${JSON.stringify(path.join(ROOT_DIR, "go"))} `; fs.writeFileSync(path.join(goDir, "go.mod"), goMod); From e98a3803bcdb0c0a601e8d56162f7975c0e50909 Mon Sep 17 00:00:00 2001 From: Zhangyi Yuan Date: Fri, 9 Oct 2026 17:44:36 +0800 Subject: [PATCH 3/3] fix: reject unclassified C# documentation build failures --- .github/workflows/sdk-dotnet.yml | 1 + scripts/docs-validation/validate.test.mjs | 42 ++++++++++++++++++++--- scripts/docs-validation/validate.ts | 6 +++- 3 files changed, 44 insertions(+), 5 deletions(-) diff --git a/.github/workflows/sdk-dotnet.yml b/.github/workflows/sdk-dotnet.yml index 63b689f94e..a4e816ab70 100644 --- a/.github/workflows/sdk-dotnet.yml +++ b/.github/workflows/sdk-dotnet.yml @@ -88,6 +88,7 @@ jobs: working-directory: ${{ inputs.sdk-home }}/scripts/docs-validation run: | npm ci + npm test npm run extract npm run validate:cs - if: failure() diff --git a/scripts/docs-validation/validate.test.mjs b/scripts/docs-validation/validate.test.mjs index 8b2859cd85..606d1b66b2 100644 --- a/scripts/docs-validation/validate.test.mjs +++ b/scripts/docs-validation/validate.test.mjs @@ -12,7 +12,7 @@ const require = createRequire(import.meta.url); const compilerDirectory = path.dirname(require.resolve("typescript/package.json")); const loader = pathToFileURL(require.resolve("tsx")).href; -function validate(context, { source = "export const value: number = 42;", types = true, compiler = true, external = false, language = "typescript" } = {}) { +function validate(context, { source = "export const value: number = 42;", types = true, compiler = true, external = false, language = "typescript", brokenProject = false } = {}) { const root = fs.mkdtempSync(path.join(os.tmpdir(), "sdk docs validation ")); context.after(() => fs.rmSync(root, { recursive: true, force: true })); const tooling = path.join(root, "scripts/docs-validation"); @@ -33,16 +33,25 @@ function validate(context, { source = "export const value: number = 42;", types } fs.copyFileSync(path.join(directory, "validate.ts"), path.join(tooling, "validate.ts")); fs.writeFileSync(path.join(root, "package.json"), JSON.stringify({ type: "module" })); + const filename = language === "csharp" ? "Sample.cs" : "sample.ts"; + if (language === "csharp") { + const sdk = path.join(root, "dotnet/src"); + fs.mkdirSync(sdk, { recursive: true }); + fs.writeFileSync(path.join(sdk, "GitHub.Copilot.SDK.csproj"), brokenProject + ? "\n" + : 'net8.0'); + fs.writeFileSync(path.join(root, "NuGet.Config"), ''); + } if (source !== null) { - fs.writeFileSync(path.join(examples, "sample.ts"), source); + fs.writeFileSync(path.join(examples, filename), source); } if (external) { fs.writeFileSync(path.join(root, "broken.ts"), 'export const value: number = "invalid";'); } fs.writeFileSync(path.join(validation, "manifest.json"), JSON.stringify({ blocks: source === null ? [] : [{ - sourceFile: "fixture.md", sourceLine: 7, outputFile: "typescript/sample.ts", + sourceFile: "fixture.md", sourceLine: 7, outputFile: `${language}/${filename}`, }] })); - const environment = { ...process.env }; + const environment = { ...process.env, DOTNET_CLI_UI_LANGUAGE: "en" }; delete environment.GITHUB_STEP_SUMMARY; const result = spawnSync(process.execPath, [ "--import", loader, path.join(tooling, "validate.ts"), `--lang=${language}`, @@ -108,3 +117,28 @@ test("generated Go module preserves a checkout path containing spaces", (context ); assert.equal(replacement.New.Path, path.join(result.root, "go")); }); + +for (const scenario of [ + { name: "valid C# succeeds", source: "public class Sample {}", status: 0 }, + { name: "C# source errors retain their documentation location", source: 'public class Sample { public int Value = "invalid"; }', status: 1, diagnostic: "CS0029" }, + { name: "C# project errors fail validation", source: "public class Sample {}", brokenProject: true, status: 1, diagnostic: "MSB4025" }, +]) { + test(scenario.name, (context) => { + const version = spawnSync("dotnet", ["--version"], { encoding: "utf8" }); + if (version.error?.code === "ENOENT") { + context.skip(".NET SDK is not installed"); + return; + } + assert.ifError(version.error); + assert.equal(version.status, 0, version.stderr); + const result = validate(context, { ...scenario, language: "csharp" }); + assert.equal(result.status, scenario.status, result.output); + if (scenario.diagnostic) { + assert.ok(result.output.includes(scenario.diagnostic), result.output); + assert.doesNotMatch(result.output, /All documentation code blocks are valid/); + } else { + assert.match(result.output, /1 files passed/); + } + if (scenario.diagnostic === "CS0029") assert.match(result.output, /fixture\.md:7/); + }); +} diff --git a/scripts/docs-validation/validate.ts b/scripts/docs-validation/validate.ts index 8e82e43b28..56da91632c 100644 --- a/scripts/docs-validation/validate.ts +++ b/scripts/docs-validation/validate.ts @@ -334,7 +334,7 @@ async function validateCSharp(): Promise { }); } } catch (err: any) { - const output = err.stdout || err.stderr || err.message || ""; + const output = [err.stdout, err.stderr].filter(Boolean).join("\n") || err.message || ""; // Parse errors by file const fileErrors = new Map(); @@ -364,6 +364,10 @@ async function validateCSharp(): Promise { errors, }); } + + if (results.every((result) => result.success)) { + throw new Error(`C# documentation compilation failed:\n${output}`); + } } return results;