diff --git a/.github/workflows/sdk-dotnet.yml b/.github/workflows/sdk-dotnet.yml index 63b689f94e..a4e816ab70 100644 --- a/.github/workflows/sdk-dotnet.yml +++ b/.github/workflows/sdk-dotnet.yml @@ -88,6 +88,7 @@ jobs: working-directory: ${{ inputs.sdk-home }}/scripts/docs-validation run: | npm ci + npm test npm run extract npm run validate:cs - if: failure() diff --git a/.github/workflows/sdk-go.yml b/.github/workflows/sdk-go.yml index e7235998f8..3e39743fe5 100644 --- a/.github/workflows/sdk-go.yml +++ b/.github/workflows/sdk-go.yml @@ -95,6 +95,7 @@ jobs: working-directory: ${{ inputs.sdk-home }}/scripts/docs-validation run: | npm ci + npm test npm run extract npm run validate:go diff --git a/.github/workflows/sdk-nodejs.yml b/.github/workflows/sdk-nodejs.yml index 9fbbf97e76..9218c9b3f7 100644 --- a/.github/workflows/sdk-nodejs.yml +++ b/.github/workflows/sdk-nodejs.yml @@ -119,6 +119,7 @@ jobs: working-directory: ${{ inputs.sdk-home }}/scripts/docs-validation run: | npm ci + npm test npm run extract npm run validate:ts diff --git a/scripts/docs-validation/package.json b/scripts/docs-validation/package.json index a7e881ba4f..82f0312d84 100644 --- a/scripts/docs-validation/package.json +++ b/scripts/docs-validation/package.json @@ -4,6 +4,7 @@ "private": true, "type": "module", "scripts": { + "test": "node --test validate.test.mjs", "extract": "tsx extract.ts", "validate": "tsx validate.ts", "validate:ts": "tsx validate.ts --lang=typescript", diff --git a/scripts/docs-validation/validate.test.mjs b/scripts/docs-validation/validate.test.mjs new file mode 100644 index 0000000000..606d1b66b2 --- /dev/null +++ b/scripts/docs-validation/validate.test.mjs @@ -0,0 +1,144 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import { createRequire } from "node:module"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; +import { fileURLToPath, pathToFileURL } from "node:url"; + +const directory = path.dirname(fileURLToPath(import.meta.url)); +const require = createRequire(import.meta.url); +const compilerDirectory = path.dirname(require.resolve("typescript/package.json")); +const loader = pathToFileURL(require.resolve("tsx")).href; + +function validate(context, { source = "export const value: number = 42;", types = true, compiler = true, external = false, language = "typescript", brokenProject = false } = {}) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "sdk docs validation ")); + context.after(() => fs.rmSync(root, { recursive: true, force: true })); + const tooling = path.join(root, "scripts/docs-validation"); + const validation = path.join(root, "docs/.validation"); + const examples = path.join(validation, language); + fs.mkdirSync(tooling, { recursive: true }); + fs.mkdirSync(examples, { recursive: true }); + fs.mkdirSync(path.join(root, "nodejs/node_modules"), { recursive: true }); + const linkType = process.platform === "win32" ? "junction" : "dir"; + fs.symlinkSync(path.join(directory, "node_modules"), path.join(tooling, "node_modules"), linkType); + if (compiler) { + fs.symlinkSync(compilerDirectory, path.join(root, "nodejs/node_modules/typescript"), linkType); + } + if (types) { + const typeDirectory = path.join(root, "node_modules/@types/node"); + fs.mkdirSync(typeDirectory, { recursive: true }); + fs.writeFileSync(path.join(typeDirectory, "index.d.ts"), "export {};\n"); + } + fs.copyFileSync(path.join(directory, "validate.ts"), path.join(tooling, "validate.ts")); + fs.writeFileSync(path.join(root, "package.json"), JSON.stringify({ type: "module" })); + const filename = language === "csharp" ? "Sample.cs" : "sample.ts"; + if (language === "csharp") { + const sdk = path.join(root, "dotnet/src"); + fs.mkdirSync(sdk, { recursive: true }); + fs.writeFileSync(path.join(sdk, "GitHub.Copilot.SDK.csproj"), brokenProject + ? "\n" + : 'net8.0'); + fs.writeFileSync(path.join(root, "NuGet.Config"), ''); + } + if (source !== null) { + fs.writeFileSync(path.join(examples, filename), source); + } + if (external) { + fs.writeFileSync(path.join(root, "broken.ts"), 'export const value: number = "invalid";'); + } + fs.writeFileSync(path.join(validation, "manifest.json"), JSON.stringify({ blocks: source === null ? [] : [{ + sourceFile: "fixture.md", sourceLine: 7, outputFile: `${language}/${filename}`, + }] })); + const environment = { ...process.env, DOTNET_CLI_UI_LANGUAGE: "en" }; + delete environment.GITHUB_STEP_SUMMARY; + const result = spawnSync(process.execPath, [ + "--import", loader, path.join(tooling, "validate.ts"), `--lang=${language}`, + ], { cwd: root, encoding: "utf8", env: environment, timeout: 60000 }); + assert.ifError(result.error); + assert.equal(result.signal, null); + return { status: result.status, output: result.stdout + result.stderr, root }; +} + +test("valid TypeScript succeeds in a path containing spaces", (context) => { + const result = validate(context); + assert.equal(result.status, 0, result.output); + assert.match(result.output, /1 files passed/); +}); + +test("example type errors fail and retain their documentation location", (context) => { + const result = validate(context, { source: 'export const value: number = "invalid";' }); + assert.equal(result.status, 1, result.output); + assert.match(result.output, /TS2322/); + assert.match(result.output, /fixture\.md:7/); +}); + +test("global compiler diagnostics fail validation", (context) => { + const result = validate(context, { types: false }); + assert.equal(result.status, 1, result.output); + assert.match(result.output, /TS2688/); + assert.doesNotMatch(result.output, /All documentation code blocks are valid/); +}); + +test("a missing compiler fails validation", (context) => { + const result = validate(context, { compiler: false }); + assert.equal(result.status, 1, result.output); + assert.match(result.output, /Cannot find module|MODULE_NOT_FOUND/); + assert.doesNotMatch(result.output, /All documentation code blocks are valid/); +}); + +test("errors outside extracted examples fail validation", (context) => { + const result = validate(context, { + source: 'export { value } from "../../../broken.js";', external: true, + }); + assert.equal(result.status, 1, result.output); + assert.match(result.output, /broken\.ts/); + assert.match(result.output, /TS2322/); +}); + +test("generated Go module preserves a checkout path containing spaces", (context) => { + const version = spawnSync("go", ["version"], { encoding: "utf8" }); + if (version.error?.code === "ENOENT") { + context.skip("Go is not installed"); + return; + } + assert.ifError(version.error); + assert.equal(version.status, 0, version.stderr); + const result = validate(context, { language: "go", source: null }); + assert.equal(result.status, 0, result.output); + const parsed = spawnSync("go", ["mod", "edit", "-json"], { + cwd: path.join(result.root, "docs/.validation/go"), encoding: "utf8", + }); + assert.ifError(parsed.error); + assert.equal(parsed.status, 0, parsed.stderr); + const replacement = JSON.parse(parsed.stdout).Replace.find( + (entry) => entry.Old.Path === "github.com/github/copilot-sdk/go", + ); + assert.equal(replacement.New.Path, path.join(result.root, "go")); +}); + +for (const scenario of [ + { name: "valid C# succeeds", source: "public class Sample {}", status: 0 }, + { name: "C# source errors retain their documentation location", source: 'public class Sample { public int Value = "invalid"; }', status: 1, diagnostic: "CS0029" }, + { name: "C# project errors fail validation", source: "public class Sample {}", brokenProject: true, status: 1, diagnostic: "MSB4025" }, +]) { + test(scenario.name, (context) => { + const version = spawnSync("dotnet", ["--version"], { encoding: "utf8" }); + if (version.error?.code === "ENOENT") { + context.skip(".NET SDK is not installed"); + return; + } + assert.ifError(version.error); + assert.equal(version.status, 0, version.stderr); + const result = validate(context, { ...scenario, language: "csharp" }); + assert.equal(result.status, scenario.status, result.output); + if (scenario.diagnostic) { + assert.ok(result.output.includes(scenario.diagnostic), result.output); + assert.doesNotMatch(result.output, /All documentation code blocks are valid/); + } else { + assert.match(result.output, /1 files passed/); + } + if (scenario.diagnostic === "CS0029") assert.match(result.output, /fixture\.md:7/); + }); +} diff --git a/scripts/docs-validation/validate.ts b/scripts/docs-validation/validate.ts index cfa14b9251..56da91632c 100644 --- a/scripts/docs-validation/validate.ts +++ b/scripts/docs-validation/validate.ts @@ -75,8 +75,8 @@ async function validateTypeScript(): Promise { try { // Run tsc - const tscPath = path.join(ROOT_DIR, "nodejs/node_modules/.bin/tsc"); - execFileSync(tscPath, ["--project", tsconfigPath], { + const tscPath = path.join(ROOT_DIR, "nodejs/node_modules/typescript/bin/tsc"); + execFileSync(process.execPath, [tscPath, "--project", tsconfigPath, "--pretty", "false"], { encoding: "utf-8", cwd: tsDir, }); @@ -98,7 +98,7 @@ async function validateTypeScript(): Promise { } } catch (err: any) { // Parse tsc output for errors - const output = err.stdout || err.stderr || err.message || ""; + const output = [err.stdout, err.stderr].filter(Boolean).join("\n") || err.message || ""; const errorLines = output.split("\n"); const fileErrors = new Map(); let currentFile = ""; @@ -106,7 +106,7 @@ async function validateTypeScript(): Promise { for (const line of errorLines) { const match = line.match(/^(.+\.ts)\((\d+),(\d+)\): error/); if (match) { - currentFile = match[1]; + currentFile = path.resolve(tsDir, match[1]); if (!fileErrors.has(currentFile)) { fileErrors.set(currentFile, []); } @@ -124,7 +124,7 @@ async function validateTypeScript(): Promise { const block = manifest.blocks.find( (b) => b.outputFile === `typescript/${file}`, ); - const errors = fileErrors.get(fullPath) || fileErrors.get(file) || []; + const errors = fileErrors.get(fullPath) || []; results.push({ file: `typescript/${file}`, @@ -134,6 +134,10 @@ async function validateTypeScript(): Promise { errors, }); } + + if (results.every((result) => result.success)) { + throw new Error(`TypeScript documentation compilation failed:\n${output}`); + } } return results; @@ -226,7 +230,7 @@ go 1.21 require github.com/github/copilot-sdk/go v0.0.0 -replace github.com/github/copilot-sdk/go => ${path.join(ROOT_DIR, "go")} +replace github.com/github/copilot-sdk/go => ${JSON.stringify(path.join(ROOT_DIR, "go"))} `; fs.writeFileSync(path.join(goDir, "go.mod"), goMod); @@ -330,7 +334,7 @@ async function validateCSharp(): Promise { }); } } catch (err: any) { - const output = err.stdout || err.stderr || err.message || ""; + const output = [err.stdout, err.stderr].filter(Boolean).join("\n") || err.message || ""; // Parse errors by file const fileErrors = new Map(); @@ -360,6 +364,10 @@ async function validateCSharp(): Promise { errors, }); } + + if (results.every((result) => result.success)) { + throw new Error(`C# documentation compilation failed:\n${output}`); + } } return results;