diff --git a/capability/app_accessibility.capability-index.json b/capability/app_accessibility.capability-index.json index 09014be2..4da73b7b 100644 --- a/capability/app_accessibility.capability-index.json +++ b/capability/app_accessibility.capability-index.json @@ -3,7 +3,7 @@ "version": "1.0", "build_id": "app_accessibility-v1.0-lists", "app_accessibility": { - "summary": "BrowserStack App Accessibility: WCAG accessibility results for mobile apps (Android and iOS), from automated App Automate tests run with accessibility checks on, and from manual App Live sessions scanned with Workflow Analyzer. Find the account's App Accessibility projects, a project's builds, and its manual reports, each with the id, status, app and issue counts needed to pick one. These capabilities list results; they do not read a build's or report's individual issues, compare builds, start tests or scans, or change or delete anything. Website accessibility (Website Scanner, Automate web tests) is not here.", + "summary": "BrowserStack App Accessibility: WCAG accessibility results for mobile apps (Android and iOS), from automated App Automate tests run with accessibility checks on, and from manual App Live sessions scanned with Workflow Analyzer. Find the account's App Accessibility projects, a project's builds, and its manual reports, each with the id, status, app and issue counts needed to pick one. These capabilities list results; they do not read a build's or report's individual issues, compare builds, start tests or scans, or change or delete anything. Websites are covered by a separate product.", "routing_terms": [ "app accessibility", "app a11y", diff --git a/capability/web_accessibility.capability-index.json b/capability/web_accessibility.capability-index.json new file mode 100644 index 00000000..3b3aa5cf --- /dev/null +++ b/capability/web_accessibility.capability-index.json @@ -0,0 +1,1274 @@ +{ + "schema_version": 1, + "version": "1.0", + "build_id": "web_accessibility-v1.0-lists", + "web_accessibility": { + "summary": "BrowserStack Web Accessibility: WCAG accessibility reports for websites and web apps. Find the account's Workflow Analyzer reports from the browser extension, Website Scanner scans with their scan history, Automated Tests projects and builds from web tests executed with accessibility checks on, and Assisted Test reports, each with the id, creator, status and issue counts needed to pick one. Each list names the id the accessibility issue tool reads. These capabilities list reports; they do not read a report's individual issues, start scans, or change or delete anything. Mobile apps are covered by a separate product.", + "routing_terms": [ + "web accessibility", + "website accessibility", + "web a11y", + "website scanner", + "site scanner", + "workflow analyzer", + "assisted test", + "automated tests accessibility" + ], + "auth": { + "type": "http", + "scheme": "basic" + }, + "base_url": "https://api-accessibility.browserstack.com", + "capabilities": [ + { + "name": "list_workflow_analyzer_reports", + "method": "GET", + "path": "/api/workflow-analyzer/v1/reports", + "mode": "read", + "entity": "workflow_analyzer_report", + "query": [ + { + "name": "limit", + "type": "integer", + "minimum": 10, + "maximum": 99, + "default": 20, + "description": "Reports per page, 10 to 99. Send the same limit with every next_page: the token carries only the page number, so a different limit shifts the pages. 100 is not offered because it returns 30 reports and says there is no next page.", + "example": 50 + }, + { + "name": "next_page", + "type": "string", + "description": "The data.pagination.nextPage token from the previous page, sent back as given. page is not read: a call sending it gets the first page every time. A changed or expired token answers 422 'Invalid pagination token'.", + "example": "eyJhbGciOiJIUzI1NiJ9.eyJwYWdlIjoyfQ.signature" + } + ], + "intent": "List the account's Workflow Analyzer reports newest first, with each report's id, name, creator, status, settings and issue counts - use this to find someone's latest reports, a report by name, or the id that reads its issues.", + "guidance": [ + "id is the workflowReportId fetchAccessibilityIssues takes to read a report's issues.", + "There is no filter by person, name or date: the list holds every report in the account, newest first. For someone's latest reports, read pages in order, with limit 99 to need fewer, and keep the rows whose createdBy.name or createdBy.id is theirs, stopping once you have enough.", + "Paging is data.pagination.hasNext and data.pagination.nextPage; send nextPage back as next_page with the same limit.", + "The settings sit under scanSettings, with wcagVersion an object of label and value, and the counts under issueSummary; the public API docs show both flat.", + "A limit outside 10 to 100 is not refused: a page of 20 comes back, and error says why.", + "These are web reports from the browser extension. A mobile app's Workflow Analyzer reports are App Accessibility manual reports, listed there." + ], + "returns": [ + "data.reports.id", + "data.reports.name", + "data.reports.createdBy", + "data.reports.status", + "data.reports.scanSettings", + "data.reports.issueSummary", + "data.pagination.hasNext", + "data.pagination.nextPage" + ], + "paginated": true, + "max_page_size": 99, + "responses": { + "200": { + "$response": "ListWorkflowAnalyzerReports" + }, + "401": { + "$response": "Unauthorized" + }, + "403": { + "$response": "Forbidden" + }, + "422": { + "$response": "InvalidPageToken" + }, + "502": { + "$response": "UpstreamFailure" + } + } + }, + { + "name": "list_website_scanner_scans", + "method": "GET", + "path": "/api/website-scanner/v1/scans", + "mode": "read", + "entity": "website_scan", + "query": [ + { + "name": "limit", + "type": "integer", + "minimum": 10, + "maximum": 25, + "default": 20, + "description": "Scans per page, 10 to 25. Above 25 the page still holds 25 scans but the next page starts past the ones in between, so they are skipped. With next_page, omit limit or send the same value: a different one answers 422.", + "example": 25 + }, + { + "name": "next_page", + "type": "string", + "description": "The data.pagination.nextPage token from the previous page, sent back as given. page is not read: a call sending it gets the first page every time. A changed or expired token answers 422 'Invalid pagination token'. page_size is not read either.", + "example": "eyJhbGciOiJIUzI1NiJ9.eyJwYWdlIjoyfQ.signature" + } + ], + "intent": "List the account's Website Scanner scans, most recently run first, with each scan's id, name, creator, dashboard link and latest run - use this to find a scan by name or person, or the scan and run ids that read its issues.", + "guidance": [ + "id is the scanId fetchAccessibilityIssues takes, and lastScanDetails.scanRunId is the scanRunId of the scan's latest run; list_website_scanner_scan_runs gives the earlier ones.", + "scanUrl is the scan's report on the Accessibility dashboard: the link to give a user.", + "There is no filter: current_user_id, team, page and page_size are not read, and every scan in the account comes back. For one person's scans keep the rows whose createdBy.name or createdBy.id is theirs.", + "Pages run from the most recently scanned, but the rows within a page are sorted by scan id, so order them by lastScanDetails.lastScanDate.", + "createdBy is an object of id and name, not the bare user id the public docs show.", + "lastScanDetails.error can hold a message while scanStatus is completed: some pages failed, and reportSummary.failure counts them." + ], + "returns": [ + "data.scans.id", + "data.scans.name", + "data.scans.createdBy", + "data.scans.scanUrl", + "data.scans.wcagVersion", + "data.scans.recurring", + "data.scans.lastScanDetails", + "data.pagination.hasNext", + "data.pagination.nextPage" + ], + "paginated": true, + "max_page_size": 25, + "responses": { + "200": { + "$response": "ListWebsiteScannerScans" + }, + "401": { + "$response": "Unauthorized" + }, + "403": { + "$response": "Forbidden" + }, + "422": { + "$response": "InvalidPageToken" + }, + "502": { + "$response": "UpstreamFailure" + } + } + }, + { + "name": "list_website_scanner_scan_runs", + "method": "GET", + "path": "/api/website-scanner/v1/scans/{scan_id}/scan_runs", + "mode": "read", + "entity": "scan_run", + "path_params": [ + { + "name": "scan_id", + "type": "integer", + "required": true, + "minimum": 1, + "description": "The scan's id, from list_website_scanner_scans or the answer of startAccessibilityScan. A scan's name is not accepted.", + "example": 4123456 + } + ], + "query": [ + { + "name": "limit", + "type": "integer", + "minimum": 10, + "maximum": 100, + "default": 20, + "description": "Runs per page, 10 to 100. With next_page, omit limit or send the same value: a different one answers 422.", + "example": 50 + }, + { + "name": "next_page", + "type": "string", + "description": "The data.pagination.nextPage token from the previous page, sent back as given. page is not read: a call sending it gets the first page every time. A changed or expired token answers 422 'Invalid pagination token'.", + "example": "eyJhbGciOiJIUzI1NiJ9.eyJwYWdlIjoyfQ.signature" + } + ], + "intent": "List one Website Scanner scan's runs newest first, with each run's id, date, issue and component counts and how many pages loaded - use this to pick an earlier run or follow a site's issues over time.", + "guidance": [ + "id is the scanRunId fetchAccessibilityIssues takes with this scan's scanId, also as compareWith.", + "Runs are ordered by scanDate, newest first. A run started by hand can carry a later id than a newer scheduled run, so do not order them by id.", + "reportSummary.failure counts the pages that did not load in that run; their issues are missing from its count.", + "An unknown scan answers 404 'Scan builds not found'." + ], + "returns": [ + "data.scanRuns.id", + "data.scanRuns.scanDate", + "data.scanRuns.issues", + "data.scanRuns.componentCount", + "data.scanRuns.reportSummary", + "data.pagination.hasNext", + "data.pagination.nextPage" + ], + "paginated": true, + "max_page_size": 100, + "responses": { + "200": { + "$response": "ListWebsiteScannerScanRuns" + }, + "401": { + "$response": "Unauthorized" + }, + "403": { + "$response": "Forbidden" + }, + "422": { + "$response": "InvalidPageToken" + }, + "404": { + "$response": "ScanNotFound" + }, + "500": { + "$response": "InternalServerError" + } + } + }, + { + "name": "list_automated_tests_projects", + "method": "GET", + "path": "/api/automated-tests/v1/projects", + "mode": "read", + "entity": "automated_tests_project", + "query": [ + { + "name": "limit", + "type": "integer", + "minimum": 10, + "maximum": 100, + "default": 10, + "description": "Projects per page, 10 to 100 (default 10). With next_page, omit limit or send the same value: a different one answers 400.", + "example": 50 + }, + { + "name": "next_page", + "type": "string", + "description": "The data.pagination.next_page token from the previous page, sent back as given. page is not read: a call sending it gets the first page every time. A changed or expired token answers 422 'Invalid pagination token'.", + "example": "eyJhbGciOiJIUzI1NiJ9.eyJwYWdlIjoyfQ.signature" + } + ], + "intent": "List the account's Automated Tests projects newest first, with each project's id, name and creation date - use this to find the project name that narrows its builds.", + "guidance": [ + "There is no name filter: name and search are not read, and the whole list comes back, newest first.", + "list_automated_tests_builds narrows to a project by its name (project_name), not by this id.", + "Paging here is snake_case: data.pagination.has_next and data.pagination.next_page.", + "Website Scanner scans are not projects here; list them with list_website_scanner_scans." + ], + "returns": [ + "data.projects.id", + "data.projects.name", + "data.projects.createdAt", + "data.pagination.has_next", + "data.pagination.next_page" + ], + "paginated": true, + "max_page_size": 100, + "responses": { + "200": { + "$response": "ListAutomatedTestsProjects" + }, + "400": { + "$response": "PagingChanged" + }, + "401": { + "$response": "Unauthorized" + }, + "403": { + "$response": "Forbidden" + }, + "422": { + "$response": "InvalidPageToken" + }, + "502": { + "$response": "UpstreamFailure" + } + } + }, + { + "name": "list_automated_tests_builds", + "method": "GET", + "path": "/api/automated-tests/v1/builds", + "mode": "read", + "entity": "automated_tests_build", + "query": [ + { + "name": "project_name", + "type": "string", + "description": "Keeps one project's builds: its full name, in any letter case. Part of a name matches nothing. Only this snake_case name is read: projectId, project_id and projectName are ignored, and every project's builds come back.", + "example": "Checkout web tests" + }, + { + "name": "limit", + "type": "integer", + "minimum": 10, + "maximum": 100, + "default": 10, + "description": "Builds per page, 10 to 100 (default 10). With next_page, omit limit or send the same value: a different one answers 400.", + "example": 50 + }, + { + "name": "next_page", + "type": "string", + "description": "The data.pagination.next_page token from the previous page, sent back as given. page is not read: a call sending it gets the first page every time. A changed or expired token answers 422 'Invalid pagination token'. Send project_name again, unchanged: a different one answers 400.", + "example": "eyJhbGciOiJIUzI1NiJ9.eyJwYWdlIjoyfQ.signature" + } + ], + "intent": "List the account's Automated Tests builds newest first, or one project's, with each build's id, number, name, creator, status and issue counts - use this to find a suite's latest run, someone's builds, or the build id that reads its issues.", + "guidance": [ + "id is the 40-character build id fetchAccessibilityIssues takes as buildId, the thBuildId in the build's dashboard link.", + "A project_name that matches no project answers 200 with data null and info ['No Projects found']: no match, not an error.", + "Builds that ran without accessibility checks are left out, so a page can hold fewer rows than limit.", + "There is no filter by person: keep the rows whose createdBy.name or createdBy.userId is theirs.", + "summary (page, component and issue counts, test health) is present only on a completed build that ran tests; a completed build with no tests reads status failed.", + "Paging here is snake_case: data.pagination.has_next and data.pagination.next_page." + ], + "returns": [ + "data.builds.id", + "data.builds.buildNumber", + "data.builds.name", + "data.builds.time", + "data.builds.createdBy", + "data.builds.projectId", + "data.builds.status", + "data.builds.wcagVersion", + "data.builds.summary", + "data.pagination.has_next", + "data.pagination.next_page", + "info" + ], + "paginated": true, + "max_page_size": 100, + "responses": { + "200": { + "$response": "ListAutomatedTestsBuilds" + }, + "400": { + "$response": "PagingChanged" + }, + "401": { + "$response": "Unauthorized" + }, + "403": { + "$response": "Forbidden" + }, + "422": { + "$response": "InvalidPageToken" + }, + "500": { + "$response": "InternalServerError" + } + } + }, + { + "name": "list_assisted_test_reports", + "method": "GET", + "path": "/api/assisted-test/v1/reports", + "mode": "read", + "entity": "assisted_test_report", + "query": [ + { + "name": "limit", + "type": "integer", + "minimum": 10, + "maximum": 99, + "default": 20, + "description": "Reports per page, 10 to 99. Send the same limit with every next_page: the token carries only the page number, so a different limit shifts the pages. 100 is not offered because it returns 30 reports and says there is no next page.", + "example": 50 + }, + { + "name": "next_page", + "type": "string", + "description": "The data.pagination.nextPage token from the previous page, sent back as given. page is not read: a call sending it gets the first page every time. A changed or expired token answers 422 'Invalid pagination token'.", + "example": "eyJhbGciOiJIUzI1NiJ9.eyJwYWdlIjoyfQ.signature" + } + ], + "intent": "List the account's Assisted Test reports newest first, with each report's id, name, creator, status, test type and issue counts - use this to find a guided keyboard, forms or other assisted check someone ran.", + "guidance": [ + "fetchAccessibilityIssues does not read Assisted Test reports: the counts here are what an agent can get, and the report itself is read on the Accessibility dashboard.", + "scanSettings.assistedTestType is a number: 0 keyboard, 1 interactive elements, 2 modals, 3 forms, 4 images, 5 layout, 6 tables, 7 responsiveness.", + "There is no filter by person, name or date: the list holds every report in the account, newest first; read pages in order, with limit 99 to need fewer, and keep the rows whose createdBy.name or createdBy.id is theirs.", + "Paging is data.pagination.hasNext and data.pagination.nextPage; send nextPage back as next_page with the same limit." + ], + "returns": [ + "data.reports.id", + "data.reports.name", + "data.reports.createdBy", + "data.reports.status", + "data.reports.scanSettings", + "data.reports.issueSummary", + "data.pagination.hasNext", + "data.pagination.nextPage" + ], + "paginated": true, + "max_page_size": 99, + "responses": { + "200": { + "$response": "ListAssistedTestReports" + }, + "401": { + "$response": "Unauthorized" + }, + "403": { + "$response": "Forbidden" + }, + "422": { + "$response": "InvalidPageToken" + }, + "502": { + "$response": "UpstreamFailure" + } + } + } + ], + "entities": { + "workflow_analyzer_report": { + "entity": "workflow_analyzer_report", + "title": "Workflow Analyzer report", + "description": "The accessibility issues the Workflow Analyzer browser extension found on the web pages or flow one person scanned.", + "aliases": [ + "workflow analyzer report", + "web workflow analyzer report", + "wa report", + "web accessibility report", + "browser extension report", + "accessibility toolkit report", + "extension scan" + ], + "id_convention": "A plain integer, discovered through the report list; it is the workflowReportId the accessibility issue tool reads.", + "parents": [], + "relations": [], + "capabilities": [ + "list_workflow_analyzer_reports" + ], + "key_facts": [ + "Reports are shared across the account, so the list holds everyone's; one person's are picked out by the creator on each row.", + "A report is created by scanning in the browser extension; there is nothing to create or delete here." + ] + }, + "website_scan": { + "entity": "website_scan", + "title": "Website scan", + "description": "A Website Scanner scan: a saved set of web page addresses and accessibility settings, run once or on a recurring basis.", + "aliases": [ + "website scan", + "website scanner", + "website scanner scan", + "site scanner", + "site scan", + "web scan", + "scan config", + "recurring scan" + ], + "id_convention": "A plain integer, discovered through the scan list or returned when a scan is started; a name is resolved to its id through that list.", + "parents": [], + "relations": [ + { + "entity": "scan_run", + "via": "runs once, or again on each recurrence" + } + ], + "capabilities": [ + "list_website_scanner_scans" + ], + "key_facts": [ + "A scan's own row carries only its latest run; the earlier runs are listed per scan.", + "Scans are shared across the account, so the list holds everyone's." + ] + }, + "scan_run": { + "entity": "scan_run", + "title": "Scan run", + "description": "One run of a Website Scanner scan, with the accessibility issues its pages had at that time.", + "aliases": [ + "scan run", + "website scanner run", + "website scanner report", + "scan report", + "previous scan" + ], + "id_convention": "A plain integer, read together with its scan's id; discovered through the scan's run list.", + "parents": [ + "website_scan" + ], + "relations": [ + { + "entity": "website_scan", + "via": "belongs to one scan" + } + ], + "capabilities": [ + "list_website_scanner_scan_runs" + ], + "key_facts": [ + "A page that did not load in a run is still counted as scanned in some places; the run's failure count says how many did not load." + ] + }, + "automated_tests_project": { + "entity": "automated_tests_project", + "title": "Automated Tests project", + "description": "A named group of web test builds that ran with accessibility checks on, usually one per test suite or site.", + "aliases": [ + "automated tests project", + "web accessibility project", + "automate accessibility project", + "sdk accessibility project" + ], + "id_convention": "A plain integer; builds are narrowed to a project by its name, not by this id.", + "parents": [], + "relations": [ + { + "entity": "automated_tests_build", + "via": "groups the builds reported under its name" + } + ], + "capabilities": [ + "list_automated_tests_projects" + ], + "key_facts": [ + "A project is created by the first test run reported under its name; there is nothing to create or delete here." + ] + }, + "automated_tests_build": { + "entity": "automated_tests_build", + "title": "Automated Tests build", + "description": "One run of web tests with accessibility checks on, through Automate or the BrowserStack SDK, and the issues its pages produced.", + "aliases": [ + "automated tests build", + "web accessibility build", + "automate accessibility build", + "sdk accessibility build", + "accessibility test run" + ], + "id_convention": "A 40-character string of lowercase letters and digits, the thBuildId in the build's dashboard link; discovered through the build list.", + "parents": [ + "automated_tests_project" + ], + "relations": [ + { + "entity": "automated_tests_project", + "via": "belongs to one project" + } + ], + "capabilities": [ + "list_automated_tests_builds" + ], + "key_facts": [ + "A build appears only if its tests ran with accessibility checks on.", + "status is the accessibility report's; a completed report whose build ran no tests reads failed." + ] + }, + "assisted_test_report": { + "entity": "assisted_test_report", + "title": "Assisted Test report", + "description": "The issues one guided Assisted Test recorded on a web page: keyboard, forms, modals, images, tables and similar checks.", + "aliases": [ + "assisted test report", + "assisted test", + "assisted testing report", + "guided accessibility test" + ], + "id_convention": "A plain integer, discovered through the report list.", + "parents": [], + "relations": [], + "capabilities": [ + "list_assisted_test_reports" + ], + "key_facts": [ + "Its individual issues are read on the Accessibility dashboard; the list gives its counts." + ] + } + }, + "schemas": { + "Creator": { + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "name": { + "type": "string" + } + } + }, + "CamelPaging": { + "type": "object", + "description": "hasNext is false on the last page, where nextPage is null.", + "properties": { + "hasNext": { + "type": "boolean" + }, + "nextPage": { + "type": "string" + } + } + }, + "SnakePaging": { + "type": "object", + "description": "has_next is false on the last page, where next_page is null.", + "properties": { + "has_next": { + "type": "boolean" + }, + "next_page": { + "type": "string" + } + } + }, + "WcagVersion": { + "type": "object", + "description": "The standard checked against.", + "properties": { + "label": { + "type": "string" + }, + "value": { + "type": "string" + } + } + }, + "IssueSummary": { + "type": "object", + "properties": { + "componentCount": { + "type": "integer" + }, + "issueCount": { + "type": "integer" + }, + "pageCount": { + "type": "integer" + } + } + }, + "EngineInfo": { + "type": "object", + "properties": { + "testEngine": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "version": { + "type": "string" + } + } + } + } + }, + "WorkflowAnalyzerReport": { + "type": "object", + "properties": { + "id": { + "type": "integer", + "description": "The workflowReportId the accessibility issue tool reads." + }, + "name": { + "type": "string" + }, + "createdBy": { + "$schema": "Creator" + }, + "status": { + "type": "string", + "description": "completed once the report is processed." + }, + "scanSettings": { + "type": "object", + "properties": { + "wcagVersion": { + "$schema": "WcagVersion" + }, + "needsReview": { + "type": "boolean" + }, + "bestPractices": { + "type": "boolean" + }, + "advanced": { + "type": "boolean" + }, + "experimental": { + "type": "boolean" + }, + "scanType": { + "type": "string", + "description": "Full page, or Part of a page." + } + } + }, + "engineInfo": { + "$schema": "EngineInfo" + }, + "issueSummary": { + "$schema": "IssueSummary" + }, + "teamId": { + "type": "integer", + "description": "Null when the report is not in a team." + } + } + }, + "AssistedTestReport": { + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "name": { + "type": "string" + }, + "createdBy": { + "$schema": "Creator" + }, + "status": { + "type": "string" + }, + "scanSettings": { + "type": "object", + "properties": { + "scanType": { + "type": "string", + "description": "Full page, or Part of a page." + }, + "assistedTestType": { + "type": "integer", + "description": "0 keyboard, 1 interactive elements, 2 modals, 3 forms, 4 images, 5 layout, 6 tables, 7 responsiveness." + } + } + }, + "engineInfo": { + "$schema": "EngineInfo" + }, + "issueSummary": { + "$schema": "IssueSummary" + }, + "teamId": { + "type": "integer", + "description": "Null when the report is not in a team." + } + } + }, + "WebsiteScan": { + "type": "object", + "properties": { + "id": { + "type": "integer", + "description": "The scanId the accessibility issue tool reads." + }, + "name": { + "type": "string" + }, + "createdBy": { + "$schema": "Creator" + }, + "teamId": { + "type": "string", + "description": "* when the scan is shared with the whole account." + }, + "wcagVersion": { + "type": "string", + "description": "A value such as wcag21aa." + }, + "needsReview": { + "type": "boolean" + }, + "bestPractices": { + "type": "boolean" + }, + "advanced": { + "type": "boolean" + }, + "status": { + "type": "string", + "description": "The latest run's report status." + }, + "recurring": { + "type": "boolean" + }, + "active": { + "type": "boolean" + }, + "nextScanDate": { + "type": "string", + "format": "date-time", + "description": "Null when the scan does not recur." + }, + "pageCount": { + "type": "integer", + "description": "How many page addresses the scan holds." + }, + "localTestingEnabled": { + "type": "boolean" + }, + "authEnabled": { + "type": "boolean" + }, + "cookiesEnabled": { + "type": "boolean" + }, + "cookieConfigIds": { + "type": "array", + "items": { + "type": "integer" + } + }, + "isCookieDeleted": { + "type": "boolean" + }, + "scanUrl": { + "type": "string", + "description": "The scan's report on the Accessibility dashboard." + }, + "lastScanDetails": { + "type": "object", + "properties": { + "scanRunId": { + "type": "integer", + "description": "The latest run's id." + }, + "lastScanDate": { + "type": "string", + "format": "date-time" + }, + "scanStatus": { + "type": "string", + "description": "completed or failed." + }, + "issues": { + "type": "integer" + }, + "reportSummary": { + "$schema": "PageLoadSummary" + }, + "error": { + "type": "string", + "description": "Null, or why pages or the run failed." + } + } + } + } + }, + "PageLoadSummary": { + "type": "object", + "description": "How the run's pages loaded.", + "properties": { + "total": { + "type": "integer" + }, + "success": { + "type": "integer" + }, + "failure": { + "type": "integer", + "description": "Pages that did not load." + }, + "redirect": { + "type": "integer" + } + } + }, + "ScanRun": { + "type": "object", + "properties": { + "id": { + "type": "integer", + "description": "The scanRunId the accessibility issue tool reads." + }, + "scanDate": { + "type": "string", + "format": "date-time" + }, + "issues": { + "type": "integer" + }, + "componentCount": { + "type": "integer" + }, + "reportSummary": { + "$schema": "PageLoadSummary" + } + } + }, + "AutomatedTestsProject": { + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "name": { + "type": "string", + "description": "What list_automated_tests_builds takes as project_name." + }, + "createdAt": { + "type": "string", + "format": "date-time" + } + } + }, + "AutomatedTestsBuild": { + "type": "object", + "properties": { + "id": { + "type": "string", + "description": "The 40-character buildId the accessibility issue tool reads." + }, + "buildNumber": { + "type": "integer" + }, + "name": { + "type": "string" + }, + "time": { + "type": "string", + "format": "date-time" + }, + "createdBy": { + "type": "object", + "properties": { + "userId": { + "type": "integer" + }, + "name": { + "type": "string" + } + } + }, + "projectId": { + "type": "integer" + }, + "status": { + "type": "string", + "description": "The accessibility report's status." + }, + "wcagVersion": { + "$schema": "WcagVersion" + }, + "needsReview": { + "type": "boolean" + }, + "bestPractice": { + "type": "boolean" + }, + "advanced": { + "type": "boolean" + }, + "summary": { + "type": "object", + "description": "Only on a completed build that ran tests.", + "properties": { + "pageCount": { + "type": "integer" + }, + "componentCount": { + "type": "integer" + }, + "issueCount": { + "type": "integer" + }, + "severityBreakdown": { + "type": "object", + "properties": { + "critical": { + "type": "integer" + }, + "serious": { + "type": "integer" + }, + "moderate": { + "type": "integer" + }, + "minor": { + "type": "integer" + } + } + }, + "health": { + "type": "object", + "description": "How the build's tests went.", + "properties": { + "total": { + "type": "integer" + }, + "passed": { + "type": "integer" + }, + "failed": { + "type": "integer" + }, + "skipped": { + "type": "integer" + }, + "pending": { + "type": "integer" + }, + "unknown": { + "type": "integer" + } + } + } + } + } + } + }, + "Failure": { + "type": "object", + "properties": { + "success": { + "type": "boolean" + }, + "data": { + "type": "object", + "description": "Null on a failure." + }, + "error": { + "type": "array", + "description": "The reasons, as text; a single string on some failures.", + "items": { + "type": "string" + } + }, + "message": { + "type": "string" + } + } + } + }, + "responses": { + "ListWorkflowAnalyzerReports": { + "description": "A page of Workflow Analyzer reports, newest first.", + "schema": { + "type": "object", + "properties": { + "success": { + "type": "boolean" + }, + "data": { + "type": "object", + "properties": { + "reports": { + "type": "array", + "items": { + "$schema": "WorkflowAnalyzerReport" + } + }, + "pagination": { + "$schema": "CamelPaging" + } + } + }, + "error": { + "type": "array", + "description": "Null, or notes such as a limit outside 10 to 100 that was replaced by the default.", + "items": { + "type": "string" + } + } + } + } + }, + "ListAssistedTestReports": { + "description": "A page of Assisted Test reports, newest first.", + "schema": { + "type": "object", + "properties": { + "success": { + "type": "boolean" + }, + "data": { + "type": "object", + "properties": { + "reports": { + "type": "array", + "items": { + "$schema": "AssistedTestReport" + } + }, + "pagination": { + "$schema": "CamelPaging" + } + } + }, + "error": { + "type": "array", + "description": "Null, or notes such as a limit outside 10 to 100 that was replaced by the default.", + "items": { + "type": "string" + } + } + } + } + }, + "ListWebsiteScannerScans": { + "description": "A page of scans, most recently run first, sorted by id within the page.", + "schema": { + "type": "object", + "properties": { + "success": { + "type": "boolean" + }, + "data": { + "type": "object", + "properties": { + "scans": { + "type": "array", + "items": { + "$schema": "WebsiteScan" + } + }, + "pagination": { + "$schema": "CamelPaging" + } + } + }, + "error": { + "type": "array", + "description": "Null, or notes such as a limit outside 10 to 100 that was replaced by the default.", + "items": { + "type": "string" + } + } + } + } + }, + "ListWebsiteScannerScanRuns": { + "description": "A page of one scan's runs, newest first.", + "schema": { + "type": "object", + "properties": { + "success": { + "type": "boolean" + }, + "data": { + "type": "object", + "properties": { + "scanRuns": { + "type": "array", + "items": { + "$schema": "ScanRun" + } + }, + "pagination": { + "$schema": "CamelPaging" + } + } + }, + "error": { + "type": "array", + "description": "Null, or notes such as a limit outside 10 to 100 that was replaced by the default.", + "items": { + "type": "string" + } + } + } + } + }, + "ListAutomatedTestsProjects": { + "description": "A page of Automated Tests projects, newest first.", + "schema": { + "type": "object", + "properties": { + "success": { + "type": "boolean" + }, + "data": { + "type": "object", + "properties": { + "projects": { + "type": "array", + "items": { + "$schema": "AutomatedTestsProject" + } + }, + "pagination": { + "$schema": "SnakePaging" + } + } + }, + "error": { + "type": "array", + "description": "Null, or notes such as a limit outside 10 to 100 that was replaced by the default.", + "items": { + "type": "string" + } + } + } + } + }, + "ListAutomatedTestsBuilds": { + "description": "A page of builds, newest first; data is null when project_name matched no project.", + "schema": { + "type": "object", + "properties": { + "success": { + "type": "boolean" + }, + "data": { + "type": "object", + "properties": { + "builds": { + "type": "array", + "items": { + "$schema": "AutomatedTestsBuild" + } + }, + "pagination": { + "$schema": "SnakePaging" + } + } + }, + "error": { + "type": "array", + "description": "Null, or notes such as a limit outside 10 to 100 that was replaced by the default.", + "items": { + "type": "string" + } + }, + "info": { + "type": "array", + "description": "['No Projects found'] when project_name matched no project.", + "items": { + "type": "string" + } + } + } + } + }, + "Unauthorized": { + "description": "The username or access key was not accepted ('Invalid credentials.'), or the plan has no API access ('You do not have access to External APIs.').", + "schema": { + "$schema": "Failure" + } + }, + "Forbidden": { + "description": "The caller's role cannot view builds and reports, or the account is restricted to another region.", + "schema": { + "$schema": "Failure" + } + }, + "InvalidPageToken": { + "description": "next_page was not a token from this list or had expired. On the Website Scanner lists, a limit that differs from the previous page also answers 422.", + "schema": { + "$schema": "Failure" + } + }, + "PagingChanged": { + "description": "limit or project_name differs from the request that produced next_page.", + "schema": { + "$schema": "Failure" + } + }, + "ScanNotFound": { + "description": "No scan with this id in the account: error is 'Scan builds not found'.", + "schema": { + "$schema": "Failure" + } + }, + "UpstreamFailure": { + "description": "The service holding the list did not answer; retry shortly.", + "schema": { + "$schema": "Failure" + } + }, + "InternalServerError": { + "description": "The list could not be read; retry shortly.", + "schema": { + "$schema": "Failure" + } + } + } + } +} diff --git a/docs/TOOLS.md b/docs/TOOLS.md index 16b8b8b9..e8233c92 100644 --- a/docs/TOOLS.md +++ b/docs/TOOLS.md @@ -348,6 +348,12 @@ searchCapability → describeCapability → invokeCapability | `list_app_accessibility_builds` | List an App Accessibility project's builds newest first, with each build's id, status, app and issue counts - use this to find the latest run, a build by name, or one person's builds. | | `list_app_accessibility_projects` | List the account's App Accessibility projects newest first, or find one by name - use this to get the project id that lists its builds. | | `list_manual_app_accessibility_reports` | List manual App Accessibility reports (App Live sessions scanned with Workflow Analyzer) newest first, with each report's id, app, status and issue counts - use this to find a report QA shared or the latest manual scan of an app. | +| `list_assisted_test_reports` | List the account's Assisted Test reports newest first, with each report's id, name, creator, status, test type and issue counts - use this to find a guided keyboard, forms or other assisted check someone ran. | +| `list_automated_tests_builds` | List the account's Automated Tests builds newest first, or one project's, with each build's id, number, name, creator, status and issue counts - use this to find a suite's latest run, someone's builds, or the build id that reads its issues. | +| `list_automated_tests_projects` | List the account's Automated Tests projects newest first, with each project's id, name and creation date - use this to find the project name that narrows its builds. | +| `list_website_scanner_scan_runs` | List one Website Scanner scan's runs newest first, with each run's id, date, issue and component counts and how many pages loaded - use this to pick an earlier run or follow a site's issues over time. | +| `list_website_scanner_scans` | List the account's Website Scanner scans, most recently run first, with each scan's id, name, creator, dashboard link and latest run - use this to find a scan by name or person, or the scan and run ids that read its issues. | +| `list_workflow_analyzer_reports` | List the account's Workflow Analyzer reports newest first, with each report's id, name, creator, status, settings and issue counts - use this to find someone's latest reports, a report by name, or the id that reads its issues. | diff --git a/src/tools/accessibility.ts b/src/tools/accessibility.ts index 2b147437..8b91c419 100644 --- a/src/tools/accessibility.ts +++ b/src/tools/accessibility.ts @@ -1678,24 +1678,28 @@ export default function addAccessibilityTools( scanId: z .string() .optional() - .describe("Website Scanner: the scan id from startAccessibilityScan."), + .describe( + "Website Scanner scan id: from startAccessibilityScan or list_website_scanner_scans (searchCapability).", + ), scanRunId: z .string() .optional() .describe( - "Website Scanner: the scan run id from startAccessibilityScan.", + "Website Scanner run id: from startAccessibilityScan, or list_website_scanner_scan_runs (searchCapability).", ), buildId: z .string() .regex(/^[a-z0-9]{40}$/, "40 lowercase letters and digits") .optional() .describe( - "Web or App Accessibility build id: the SDK's testhub id, a web link's thBuildId, or an app build via searchCapability.", + "Web or App Accessibility build id: the SDK's testhub id, a link's thBuildId, or one listed via searchCapability.", ), workflowReportId: z .string() .optional() - .describe("Workflow Analyzer: the report's numeric id."), + .describe( + "Workflow Analyzer report id: from its link, or list_workflow_analyzer_reports (searchCapability).", + ), appReportId: z .string() .regex( diff --git a/src/tools/capability-registry/resolve.ts b/src/tools/capability-registry/resolve.ts index 98e123ef..c8fd460e 100644 --- a/src/tools/capability-registry/resolve.ts +++ b/src/tools/capability-registry/resolve.ts @@ -27,9 +27,10 @@ export interface InvokeResult { /** * Whether this response is the whole answer. * - * False when the envelope itself says there is another page (`info.next`), so a caller - * knows to ask for one rather than assuming it has everything. This is a peek at one - * declared field, not a reshaping of the body. + * False when the envelope itself says there is another page (`info.next`, or + * `data.pagination.hasNext` / `has_next`), so a caller knows to ask for one rather than + * assuming it has everything. This is a peek at one declared field, not a reshaping of + * the body. */ completed: boolean; /** The product's status, and its body exactly as sent. */ @@ -42,12 +43,18 @@ export interface InvokeResult { } function hasNextPage(body: unknown): boolean { - if (typeof body !== "object" || body === null || Array.isArray(body)) - return false; - const info = (body as Record).info; - if (typeof info !== "object" || info === null) return false; - const next = (info as Record).next; - return next !== null && next !== undefined && next !== false; + const record = (value: unknown): Record | undefined => + typeof value === "object" && value !== null && !Array.isArray(value) + ? (value as Record) + : undefined; + const envelope = record(body); + if (!envelope) return false; + const next = record(envelope.info)?.next; + if (next !== null && next !== undefined && next !== false) return true; + // Accessibility's lists page under data.pagination instead, camelCase on some and + // snake_case on others. + const pagination = record(record(envelope.data)?.pagination); + return pagination?.hasNext === true || pagination?.has_next === true; } /** diff --git a/src/tools/tool-handoff.ts b/src/tools/tool-handoff.ts index 6f6a23fa..f4a5e2d7 100644 --- a/src/tools/tool-handoff.ts +++ b/src/tools/tool-handoff.ts @@ -91,12 +91,14 @@ export const NEEDS_SESSION_ID = /** * A scan run's ids come from startAccessibilityScan; a build's id is printed by the SDK at * the end of its run. The tool takes a report's dashboard link, or an Automate link, and - * looks the ids up itself; for a past run, Ask AI can locate it. App Accessibility's builds - * and manual reports are listed by its capabilities, which the id arguments name, as this - * line has no room left within the description budget. + * looks the ids up itself. For a past report, the web_accessibility capabilities list + * Workflow Analyzer reports, scans and their runs, and Automated Tests builds with the id + * each takes; App Accessibility's builds and manual reports are listed by its own + * capabilities, which the id arguments name, as this line has no room left within the + * description budget. */ export const NEEDS_A11Y_SCAN_ID = - ' Ids: startAccessibilityScan, a link, or askBrowserStackAI with product "a11y".'; + " Ids: a link, startAccessibilityScan, or web_accessibility lists (searchCapability)."; /** Auth-config ids are not listable by any tool here; Ask AI can find one. */ export const NEEDS_A11Y_CONFIG_ID = diff --git a/tests/tools/askBrowserstackE2E.test.ts b/tests/tools/askBrowserstackE2E.test.ts index 9c49e44f..b1f766b2 100644 --- a/tests/tools/askBrowserstackE2E.test.ts +++ b/tests/tools/askBrowserstackE2E.test.ts @@ -309,7 +309,10 @@ describe("askBrowserStackAI, end to end through the server factory", () => { const productText = shape?.product?.description as string; for (const value of values) expect(productText).toContain(`${value} (`); - for (const handoff of [NEEDS_A11Y_SCAN_ID, NEEDS_A11Y_CONFIG_ID]) { + // Report ids are listed by the web_accessibility capabilities now, so that hand-off + // no longer sends the model to Ask AI; the auth-config one still does. + expect(NEEDS_A11Y_SCAN_ID).not.toMatch(/askBrowserStackAI/); + for (const handoff of [NEEDS_A11Y_CONFIG_ID]) { const named = handoff.match(/askBrowserStackAI with product "([^"]+)"/)?.[1]; expect(named).toBeDefined(); expect(values).toContain(named); diff --git a/tests/tools/capabilityAppAccessibility.test.ts b/tests/tools/capabilityAppAccessibility.test.ts index 7d7d2741..1be73a5c 100644 --- a/tests/tools/capabilityAppAccessibility.test.ts +++ b/tests/tools/capabilityAppAccessibility.test.ts @@ -75,7 +75,7 @@ describe("the App Accessibility capability index", () => { it.each([ ["List my App Accessibility reports"], - ["Show the Workflow Analyzer reports QA ran last week"], + ["Show the manual app accessibility reports QA ran last week"], ["What builds does my app accessibility project have?"], ])("routes %j to App Accessibility without asking", (query) => { expect(ambiguousProducts(products, query).settled).toBe( diff --git a/tests/tools/capabilityRegistry.test.ts b/tests/tools/capabilityRegistry.test.ts index 284cba2f..20c7f795 100644 --- a/tests/tools/capabilityRegistry.test.ts +++ b/tests/tools/capabilityRegistry.test.ts @@ -458,6 +458,22 @@ describe("invoke — one request, response returned untouched", () => { expect(result.completed).toBe(false); }); + // Accessibility's lists page with a token under data.pagination (camelCase on some lists, + // snake_case on others) rather than info.next. Read as complete, an agent would stop at + // the first 20 reports and answer "the last report Priya ran" from the wrong page. + it.each([ + [{ success: true, data: { reports: [], pagination: { hasNext: true, nextPage: "t" } } }, false], + [{ success: true, data: { builds: [], pagination: { has_next: true, next_page: "t" } } }, false], + [{ success: true, data: { scans: [], pagination: { hasNext: false, nextPage: null } } }, true], + [{ success: true, data: null, info: ["No Projects found"] }, true], + ])("reads another page from a data.pagination envelope (%#)", async (body, completed) => { + const result = await invoke( + LIST_CASES, { path_params: { project_id: 1, folder_id: 2 } }, + "https://a11y.example", credentials, async () => ({ status: 200, body }), + ); + expect(result.completed).toBe(completed); + }); + it("mirrors a non-2xx and lets the product's own body explain it", async () => { const transport = async () => ({ status: 422, diff --git a/tests/tools/capabilityWebAccessibility.test.ts b/tests/tools/capabilityWebAccessibility.test.ts new file mode 100644 index 00000000..ef712b4a --- /dev/null +++ b/tests/tools/capabilityWebAccessibility.test.ts @@ -0,0 +1,167 @@ +import { describe, expect, it } from "vitest"; +import { readdirSync, readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; + +import { + ambiguousProducts, + searchCapabilities, +} from "../../src/tools/capability-registry/search.js"; + +// Web Accessibility's lists (Workflow Analyzer reports, Website Scanner scans and their runs, +// Automated Tests projects and builds, Assisted Test reports) are reached through the +// capability registry. Without them an agent asked for "the last 3 reports Priya ran" had +// no list to read and handed the question to Ask AI. What has to hold is that each list +// names the id fetchAccessibilityIssues reads, that the parameters the API ignores are not +// offered, and that the words users say reach the right product. + +const DIR = new URL("../../capability/", import.meta.url); + +function shipped(): Record { + const products: Record = {}; + for (const file of readdirSync(fileURLToPath(DIR))) { + if (!file.endsWith(".capability-index.json")) continue; + const raw = JSON.parse( + readFileSync(fileURLToPath(new URL(file, DIR)), "utf8"), + ); + const [name] = Object.keys(raw).filter( + (k) => !["schema_version", "version", "build_id"].includes(k), + ); + products[name] = raw[name]; + } + return products; +} + +const products = shipped(); +const web = products.web_accessibility; +const capability = (name: string) => + web.capabilities.find((c: any) => c.name === name); +const queryNames = (name: string) => + (capability(name).query ?? []).map((q: any) => q.name); + +describe("the Web Accessibility capability index", () => { + it("ships the six lists, as paged reads, each on an entity that names it", () => { + const names = web.capabilities.map((c: any) => c.name).sort(); + expect(names).toEqual([ + "list_assisted_test_reports", + "list_automated_tests_builds", + "list_automated_tests_projects", + "list_website_scanner_scan_runs", + "list_website_scanner_scans", + "list_workflow_analyzer_reports", + ]); + for (const c of web.capabilities) { + expect(c.mode).toBe("read"); + expect(c.paginated).toBe(true); + expect(c.max_page_size).toBeGreaterThan(0); + expect(web.entities[c.entity].capabilities).toContain(c.name); + } + for (const entity of Object.values(web.entities)) { + expect(entity.description.length).toBeLessThanOrEqual(140); + } + expect(web.summary.length).toBeLessThanOrEqual(900); + }); + + it("caps each page size where paging still works", () => { + const limit = (name: string) => + capability(name).query.find((q: any) => q.name === "limit"); + // 100 returns 30 reports and says there is no next page. + expect(limit("list_workflow_analyzer_reports").maximum).toBe(99); + expect(limit("list_assisted_test_reports").maximum).toBe(99); + // Above 25 the page holds 25 scans and the next one starts past the ones in between. + expect(limit("list_website_scanner_scans").maximum).toBe(25); + for (const c of web.capabilities) { + expect(c.max_page_size).toBe( + c.query.find((q: any) => q.name === "limit").maximum, + ); + } + }); + + it("says a changed limit is refused only by the lists that refuse it", () => { + // All six answer a bad next_page with 422 (InvalidPageToken). Only the Website Scanner + // lists also answer a changed limit with 422; the Automated Tests lists answer it with + // 400 (PagingChanged), and the Workflow Analyzer and Assisted Test lists do not check + // it, so their pages shift instead. + const invalidToken = web.responses.InvalidPageToken.description; + expect(invalidToken).toMatch(/Website Scanner lists/); + expect(invalidToken).not.toMatch(/^[^.]*limit changed/); + for (const name of [ + "list_automated_tests_projects", + "list_automated_tests_builds", + ]) { + expect(capability(name).responses["400"]).toEqual({ + $response: "PagingChanged", + }); + } + }); + + it("does not offer the parameters the API ignores", () => { + // The public docs list these; the handlers never read them, so each would return the + // whole account's list while looking filtered. + for (const ignored of ["current_user_id", "team", "page", "page_size"]) { + expect(queryNames("list_website_scanner_scans")).not.toContain(ignored); + } + for (const ignored of ["projectId", "projectName", "project_id"]) { + expect(queryNames("list_automated_tests_builds")).not.toContain(ignored); + } + expect(queryNames("list_automated_tests_builds")).toContain("project_name"); + }); + + it("says which field each list hands to fetchAccessibilityIssues", () => { + const guidance = (name: string) => capability(name).guidance.join(" "); + expect(guidance("list_workflow_analyzer_reports")).toMatch( + /workflowReportId/, + ); + expect(guidance("list_website_scanner_scans")).toMatch(/scanId/); + expect(guidance("list_website_scanner_scans")).toMatch(/scanRunId/); + expect(guidance("list_website_scanner_scan_runs")).toMatch(/scanRunId/); + expect(guidance("list_automated_tests_builds")).toMatch(/buildId/); + // Assisted Test reports have no issue reader here, and saying so stops a wasted call. + expect(guidance("list_assisted_test_reports")).toMatch( + /fetchAccessibilityIssues does not read/, + ); + }); + + it.each([ + ["List my website scanner scans"], + ["Show the builds of my automated tests project"], + ["Find the last 3 web workflow analyzer reports Priya ran"], + ["What did the latest scan run of our website scan find?"], + ])("routes %j to Web Accessibility", (query) => { + expect(ambiguousProducts(products, query).settled).toBe( + "web_accessibility", + ); + }); + + it("asks web or app when only Workflow Analyzer is named, as both products have it", () => { + const { products: asked } = ambiguousProducts( + products, + "last 3 workflow analyzer reports by Priya", + ); + expect(asked).toEqual( + expect.arrayContaining(["app_accessibility", "web_accessibility"]), + ); + }); + + it("leaves App Accessibility and Test Reporting questions where they were", () => { + expect( + ambiguousProducts(products, "List my App Accessibility reports").settled, + ).toBe("app_accessibility"); + expect( + ambiguousProducts(products, "Which tests are flaky in my build?").settled, + ).toBe("tra"); + }); + + it.each([ + ["list workflow analyzer reports", "list_workflow_analyzer_reports"], + ["list website scanner scans", "list_website_scanner_scans"], + ["list the runs of a website scanner scan", "list_website_scanner_scan_runs"], + ["list automated tests projects", "list_automated_tests_projects"], + ["list the builds of an automated tests project", "list_automated_tests_builds"], + ["list assisted test reports", "list_assisted_test_reports"], + ])("finds %j as %s", (query, expected) => { + const { hits } = searchCapabilities(products, query, { + product: "web_accessibility", + }); + expect(hits[0]?.capability.name).toBe(expected); + }); +});