From a478fd46e639269ec550d6992109e111e13e3217 Mon Sep 17 00:00:00 2001 From: hpoeche Date: Thu, 30 Jul 2026 10:25:58 +0200 Subject: [PATCH 01/14] First draft of applying testdatagen --- sdk/test/testdatagen/Readme.md | 16 +++++ sdk/test/testdatagen/example_tests.py | 93 +++++++++++++++++++++++++++ 2 files changed, 109 insertions(+) create mode 100644 sdk/test/testdatagen/Readme.md create mode 100644 sdk/test/testdatagen/example_tests.py diff --git a/sdk/test/testdatagen/Readme.md b/sdk/test/testdatagen/Readme.md new file mode 100644 index 000000000..827ffcbe9 --- /dev/null +++ b/sdk/test/testdatagen/Readme.md @@ -0,0 +1,16 @@ +## Fuzzy Testing with `aas-core-testdatagen` +This folder contains a script to test the implementation of the `basyx-python-sdk` against the example files +generated by the [`aas-core-works/aas-core-testdatagen`](https://github.com/aas-core-works/aas-core-testdatagen) project. + +### Prerequisites +The script requires the example files to exist in the specified path in the same structure as they are shipped in the +[`aas-specs-metamodel`](https://github.com/admin-shell-io/aas-specs-metamodel) repository. Specifically, if +`$EXAMPLES` is the path passed to the script the following directories are scanned for `.json` and `.xml` files respectively: + +``` +$EXAMPLES/json/examples/generated +$EXAMPLES/xml/examples/generated +``` + +Additionally, the `basyx-python-sdk` must be installed via `pip install sdk/.`. + diff --git a/sdk/test/testdatagen/example_tests.py b/sdk/test/testdatagen/example_tests.py new file mode 100644 index 000000000..cecfe1f49 --- /dev/null +++ b/sdk/test/testdatagen/example_tests.py @@ -0,0 +1,93 @@ +import argparse +import logging +import re +import sys +import traceback +from pathlib import Path +from typing import Callable, Literal, Optional + +from basyx.aas import adapter +from basyx.aas.model import AASConstraintViolation + +logger = logging.getLogger(__name__) + +def sanitize_name(name: str, max_len: int = 120) -> str: + name = re.sub(r'[<>:"/\\|?*\x00-\x1f\s]', '_', name) + return name[:max_len] + +def test_json_example(example_file: Path, base_path: Optional[Path] = None, output_dir: Optional[Path] = None) -> bool: + """ + Attempts to read and deserialize an example file. Reports the status and saves information to output path. + + :param example_file: File which contains the example that should be deserialized + :param deserializer: Function that is used to deserialize the example + :param output_dir: Path to store the failed outputs + :return: bool that indicates if the example was successfully deserialized + """ + try: + adapter.json.read_aas_json_file(example_file, failsafe=False) + return True + except (KeyError, TypeError, ValueError, AASConstraintViolation) as ex: + tb = traceback.format_exc() + error_msg = str(ex).split(">>>")[0].strip() + + rel_path = example_file.relative_to(base_path) if base_path else example_file + logger.error(f"[JSON] ERROR on {rel_path}: {error_msg}") + + if output_dir: + error_dir = output_dir / sanitize_name(error_msg) + error_dir.mkdir(parents=True, exist_ok=True) + error_file = error_dir / f"{sanitize_name(example_file.stem)}.txt" + + json_content = example_file.read_text(encoding="utf-8") + error_output = f"=== JSON File: {rel_path} ===\n{json_content}\n\n=== Stacktrace ====\n{tb}" + error_file.write_text(error_output, encoding="utf-8") + return False + except NotImplementedError as ex: + tb = traceback.format_exc() + error_msg = str(ex).split(">>>")[0].strip() + + rel_path = example_file.relative_to(base_path) if base_path else example_file + logger.warning(f"[JSON] Unimplemented behavior on {rel_path}: {error_msg}") + + if output_dir: + error_dir = output_dir / "NotImplementedError" / sanitize_name(error_msg) + error_dir.mkdir(parents=True, exist_ok=True) + error_file = error_dir / f"{sanitize_name(example_file.stem)}.txt" + + json_content = example_file.read_text(encoding="utf-8") + error_output = f"=== JSON File: {rel_path} ===\n{json_content}\n\n=== Stacktrace ====\n{tb}" + error_file.write_text(error_output, encoding="utf-8") + return True + +def main(example_path_str: str, output_path_str: Optional[str])-> None: + example_path = Path(example_path_str) + json_example_dir = example_path / "json" / "examples" + xml_example_dir = example_path / "xml" / "examples" + + output_path = None + if output_path_str: + output_path = Path(output_path_str) + output_path.mkdir(parents=True, exist_ok=True) + + total = 0 + failed = 0 + for json_test in json_example_dir.glob("**/*.json"): + total += 1 + success = test_json_example(json_test, json_example_dir, output_path) + if not success: + failed += 1 + + if failed == 0: + logger.info(f"No JSON tests out of {total:d} failed") + else: + logger.error(f"Failed {failed:d}/{total:d} json tests") + sys.exit(failed) + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument("--examples", required=True, help="path to examples directory") + parser.add_argument("--output", required=False, help="path to output directory") + args = parser.parse_args() + + main(args.examples, args.output) From 5bb304016a312cbb8f874c9be7c77242b1b652c4 Mon Sep 17 00:00:00 2001 From: hpoeche Date: Thu, 30 Jul 2026 12:45:39 +0200 Subject: [PATCH 02/14] Draft: only basic JSON test handling For now only the JSON examples are considered. Errors are sorted into three categories: Errors that occur from deserialization, NotImplementedErrors that are explicit developer choices and Unexpected errors to cover future errors of the sdk. --- sdk/test/testdatagen/example_tests.py | 31 ++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/sdk/test/testdatagen/example_tests.py b/sdk/test/testdatagen/example_tests.py index cecfe1f49..6afdc0da5 100644 --- a/sdk/test/testdatagen/example_tests.py +++ b/sdk/test/testdatagen/example_tests.py @@ -37,7 +37,9 @@ def test_json_example(example_file: Path, base_path: Optional[Path] = None, outp if output_dir: error_dir = output_dir / sanitize_name(error_msg) error_dir.mkdir(parents=True, exist_ok=True) - error_file = error_dir / f"{sanitize_name(example_file.stem)}.txt" + # Use the full relative path (not just the stem) to avoid collisions: generated example sets reuse + # the same filenames (e.g. "fuzzed_01.json") across many different subdirectories. + error_file = error_dir / f"{sanitize_name(str(rel_path))}.txt" json_content = example_file.read_text(encoding="utf-8") error_output = f"=== JSON File: {rel_path} ===\n{json_content}\n\n=== Stacktrace ====\n{tb}" @@ -48,17 +50,40 @@ def test_json_example(example_file: Path, base_path: Optional[Path] = None, outp error_msg = str(ex).split(">>>")[0].strip() rel_path = example_file.relative_to(base_path) if base_path else example_file - logger.warning(f"[JSON] Unimplemented behavior on {rel_path}: {error_msg}") + logger.warning(f"[JSON] NOT_IMPLEMENTED on {rel_path}: {error_msg}") if output_dir: error_dir = output_dir / "NotImplementedError" / sanitize_name(error_msg) error_dir.mkdir(parents=True, exist_ok=True) - error_file = error_dir / f"{sanitize_name(example_file.stem)}.txt" + # Use the full relative path (not just the stem) to avoid collisions: generated example sets reuse + # the same filenames (e.g. "fuzzed_01.json") across many different subdirectories. + error_file = error_dir / f"{sanitize_name(str(rel_path))}.txt" json_content = example_file.read_text(encoding="utf-8") error_output = f"=== JSON File: {rel_path} ===\n{json_content}\n\n=== Stacktrace ====\n{tb}" error_file.write_text(error_output, encoding="utf-8") return True + except Exception as ex: + # Catch-all so an exception type not (yet) anticipated by the except clauses above (e.g. thrown by a + # future implementation change) fails just this one example instead of aborting the whole run. + # Deliberately not `except BaseException`, so KeyboardInterrupt/SystemExit still propagate. + tb = traceback.format_exc() + error_msg = str(ex).split(">>>")[0].strip() + + rel_path = example_file.relative_to(base_path) if base_path else example_file + logger.error(f"[JSON] UNEXPECTED {type(ex).__name__} on {rel_path}: {error_msg}") + + if output_dir: + error_dir = output_dir / "UnexpectedError" / sanitize_name(type(ex).__name__) / sanitize_name(error_msg) + error_dir.mkdir(parents=True, exist_ok=True) + # Use the full relative path (not just the stem) to avoid collisions: generated example sets reuse + # the same filenames (e.g. "fuzzed_01.json") across many different subdirectories. + error_file = error_dir / f"{sanitize_name(str(rel_path))}.txt" + + json_content = example_file.read_text(encoding="utf-8") + error_output = f"=== JSON File: {rel_path} ===\n{json_content}\n\n=== Stacktrace ====\n{tb}" + error_file.write_text(error_output, encoding="utf-8") + return False def main(example_path_str: str, output_path_str: Optional[str])-> None: example_path = Path(example_path_str) From e7fc1a0f9b90c7ebd6462022cf05c1ba76ccef0a Mon Sep 17 00:00:00 2001 From: paul-gerber-svg Date: Thu, 30 Jul 2026 15:21:40 +0200 Subject: [PATCH 03/14] Add XML testing and refactor test logic into a generic runner function toeliminate code duplication --- sdk/test/testdatagen/example_tests.py | 186 ++++++++++++++++---------- 1 file changed, 115 insertions(+), 71 deletions(-) diff --git a/sdk/test/testdatagen/example_tests.py b/sdk/test/testdatagen/example_tests.py index 6afdc0da5..b13ee166d 100644 --- a/sdk/test/testdatagen/example_tests.py +++ b/sdk/test/testdatagen/example_tests.py @@ -4,115 +4,159 @@ import sys import traceback from pathlib import Path -from typing import Callable, Literal, Optional +from typing import Optional from basyx.aas import adapter from basyx.aas.model import AASConstraintViolation logger = logging.getLogger(__name__) + def sanitize_name(name: str, max_len: int = 120) -> str: - name = re.sub(r'[<>:"/\\|?*\x00-\x1f\s]', '_', name) - return name[:max_len] + pattern = r'[<>:"/\\|?*\x00-\x1f\s\'{}\(\)\[\]]' + name = re.sub(pattern, '_', name) + name = re.sub(r'_+', '_', name) + return name.strip('_')[:max_len] -def test_json_example(example_file: Path, base_path: Optional[Path] = None, output_dir: Optional[Path] = None) -> bool: - """ - Attempts to read and deserialize an example file. Reports the status and saves information to output path. - :param example_file: File which contains the example that should be deserialized - :param deserializer: Function that is used to deserialize the example - :param output_dir: Path to store the failed outputs - :return: bool that indicates if the example was successfully deserialized - """ + +def write_error_report(file_type, example_file, rel_path, error_dir, tb): + error_dir.mkdir(parents=True, exist_ok=True) + # Use the full relative path (not just the stem) to avoid collisions: generated example sets reuse + # the same filenames across many different subdirectories. + error_file = error_dir / f"{sanitize_name(str(rel_path))}.txt" + + content = example_file.read_text(encoding="utf-8") + error_output = f"=== {file_type.upper()} File: {rel_path} ===\n{content}\n\n=== Stacktrace ====\n{tb}" + error_file.write_text(error_output, encoding="utf-8") + + +def run_example_test(file_type, example_file, read_fn, base_path=None, output_dir=None, is_xml=False): + rel_path = example_file.relative_to(base_path) if base_path else example_file + try: - adapter.json.read_aas_json_file(example_file, failsafe=False) + read_fn(example_file) return True except (KeyError, TypeError, ValueError, AASConstraintViolation) as ex: tb = traceback.format_exc() - error_msg = str(ex).split(">>>")[0].strip() - - rel_path = example_file.relative_to(base_path) if base_path else example_file - logger.error(f"[JSON] ERROR on {rel_path}: {error_msg}") + error_msg = extract_error_message(ex) if is_xml else str(ex).split(">>>")[0].strip() + logger.error(f"[{file_type.upper()}] ERROR on {rel_path}: {error_msg}") if output_dir: - error_dir = output_dir / sanitize_name(error_msg) - error_dir.mkdir(parents=True, exist_ok=True) - # Use the full relative path (not just the stem) to avoid collisions: generated example sets reuse - # the same filenames (e.g. "fuzzed_01.json") across many different subdirectories. - error_file = error_dir / f"{sanitize_name(str(rel_path))}.txt" - - json_content = example_file.read_text(encoding="utf-8") - error_output = f"=== JSON File: {rel_path} ===\n{json_content}\n\n=== Stacktrace ====\n{tb}" - error_file.write_text(error_output, encoding="utf-8") + error_dir = output_dir / file_type / sanitize_name(error_msg) + write_error_report(file_type, example_file, rel_path, error_dir, tb) return False + except NotImplementedError as ex: tb = traceback.format_exc() - error_msg = str(ex).split(">>>")[0].strip() - - rel_path = example_file.relative_to(base_path) if base_path else example_file - logger.warning(f"[JSON] NOT_IMPLEMENTED on {rel_path}: {error_msg}") + error_msg = extract_error_message(ex) if is_xml else str(ex).split(">>>")[0].strip() + logger.warning(f"[{file_type.upper()}] NOT_IMPLEMENTED on {rel_path}: {error_msg}") if output_dir: - error_dir = output_dir / "NotImplementedError" / sanitize_name(error_msg) - error_dir.mkdir(parents=True, exist_ok=True) - # Use the full relative path (not just the stem) to avoid collisions: generated example sets reuse - # the same filenames (e.g. "fuzzed_01.json") across many different subdirectories. - error_file = error_dir / f"{sanitize_name(str(rel_path))}.txt" - - json_content = example_file.read_text(encoding="utf-8") - error_output = f"=== JSON File: {rel_path} ===\n{json_content}\n\n=== Stacktrace ====\n{tb}" - error_file.write_text(error_output, encoding="utf-8") + error_dir = output_dir / file_type / "NotImplementedError" / sanitize_name(error_msg) + write_error_report(file_type, example_file, rel_path, error_dir, tb) return True + except Exception as ex: # Catch-all so an exception type not (yet) anticipated by the except clauses above (e.g. thrown by a # future implementation change) fails just this one example instead of aborting the whole run. # Deliberately not `except BaseException`, so KeyboardInterrupt/SystemExit still propagate. tb = traceback.format_exc() - error_msg = str(ex).split(">>>")[0].strip() - - rel_path = example_file.relative_to(base_path) if base_path else example_file - logger.error(f"[JSON] UNEXPECTED {type(ex).__name__} on {rel_path}: {error_msg}") + error_msg = extract_error_message(ex) if is_xml else str(ex).split(">>>")[0].strip() + logger.error(f"[{file_type.upper()}] UNEXPECTED {type(ex).__name__} on {rel_path}: {error_msg}") if output_dir: - error_dir = output_dir / "UnexpectedError" / sanitize_name(type(ex).__name__) / sanitize_name(error_msg) - error_dir.mkdir(parents=True, exist_ok=True) - # Use the full relative path (not just the stem) to avoid collisions: generated example sets reuse - # the same filenames (e.g. "fuzzed_01.json") across many different subdirectories. - error_file = error_dir / f"{sanitize_name(str(rel_path))}.txt" - - json_content = example_file.read_text(encoding="utf-8") - error_output = f"=== JSON File: {rel_path} ===\n{json_content}\n\n=== Stacktrace ====\n{tb}" - error_file.write_text(error_output, encoding="utf-8") + error_dir = ( + output_dir + / file_type + / "UnexpectedError" + / sanitize_name(type(ex).__name__) + / sanitize_name(error_msg) + ) + write_error_report(file_type, example_file, rel_path, error_dir, tb) return False -def main(example_path_str: str, output_path_str: Optional[str])-> None: - example_path = Path(example_path_str) - json_example_dir = example_path / "json" / "examples" - xml_example_dir = example_path / "xml" / "examples" +def extract_error_message(ex): + message = str(ex) + cause = ex.__cause__ + while cause is not None: + message = f"{message} -> {cause}" + cause = cause.__cause__ + return re.sub(r'on line [0-9]+', "", message.split("->")[-1]).strip() + - output_path = None - if output_path_str: - output_path = Path(output_path_str) +def test_json_example(example_file: Path, base_path: Optional[Path] = None, output_dir: Optional[Path] = None) -> bool: + """ + Attempts to read and deserialize an example file. Reports the status and saves information to output path. + + :param example_file: File which contains the example that should be deserialized + :param base_path: Base directory to compute relative paths + :param output_dir: Path to store the failed outputs + :return: bool that indicates if the example was successfully deserialized + """ + return run_example_test( + file_type="json", + example_file=example_file, + read_fn=lambda f: adapter.json.read_aas_json_file(f, failsafe=False), + base_path=base_path, + output_dir=output_dir, + ) + + +def test_xml_example(example_file: Path, base_path: Optional[Path] = None, output_dir: Optional[Path] = None) -> bool: + """ + Attempts to read and deserialize an XML example file. Reports the status and saves information to output path. + + :param example_file: File which contains the example that should be deserialized + :param base_path: Base directory to compute relative paths + :param output_dir: Path to store the failed outputs + :return: bool that indicates if the example was successfully deserialized + """ + return run_example_test( + file_type="xml", + example_file=example_file, + read_fn=lambda f: adapter.xml.read_aas_xml_file(f, failsafe=False), + base_path=base_path, + output_dir=output_dir, + is_xml=True, + ) + + +def main(example_path_str: str, output_path_str: Optional[str]) -> None: + example_path = Path(example_path_str) + output_path = Path(output_path_str) if output_path_str else None + if output_path: output_path.mkdir(parents=True, exist_ok=True) - total = 0 - failed = 0 - for json_test in json_example_dir.glob("**/*.json"): - total += 1 - success = test_json_example(json_test, json_example_dir, output_path) - if not success: - failed += 1 + test_configs = [ + ("JSON", example_path / "json" / "examples" / "generated", "*.json", test_json_example), + ("XML", example_path / "xml" / "examples" / "generated", "*.xml", test_xml_example), + ] + + total_failed = 0 + for name, directory, glob_pattern, test_fn in test_configs: + total = 0 + failed = 0 + for test_file in sorted(directory.glob(f"**/{glob_pattern}")): + total += 1 + if not test_fn(test_file, directory, output_path): + failed += 1 + + if failed == 0: + logger.info(f"No {name} tests out of {total:d} failed") + else: + logger.error(f"Failed {failed:d}/{total:d} {name.lower()} tests") + + total_failed += failed + + sys.exit(total_failed) - if failed == 0: - logger.info(f"No JSON tests out of {total:d} failed") - else: - logger.error(f"Failed {failed:d}/{total:d} json tests") - sys.exit(failed) if __name__ == '__main__': + logging.basicConfig(level=logging.INFO) parser = argparse.ArgumentParser() parser.add_argument("--examples", required=True, help="path to examples directory") parser.add_argument("--output", required=False, help="path to output directory") args = parser.parse_args() - main(args.examples, args.output) + main(args.examples, args.output) \ No newline at end of file From e2e1905bacab40642d41417c34fb401fb21e388e Mon Sep 17 00:00:00 2001 From: hpoeche Date: Sat, 1 Aug 2026 16:22:12 +0200 Subject: [PATCH 04/14] Add sdk-testdatagen compsite action We add a composite github action `sdk-test-testdatagen` that fetches the example files from the `aas-specs-metmodel` repository in the correct verion, runs the `example_tests.py` script and uploads the output as an artifact to github. For testing this action is temporarily called in the ci, that runs on every pull_request. This should be changed to only run on PRs into main before this PR is approved. --- .../actions/sdk-test-testdatagen/action.yml | 33 +++++++++++++++++++ .github/workflows/ci.yml | 16 +++++++++ 2 files changed, 49 insertions(+) create mode 100644 .github/actions/sdk-test-testdatagen/action.yml diff --git a/.github/actions/sdk-test-testdatagen/action.yml b/.github/actions/sdk-test-testdatagen/action.yml new file mode 100644 index 000000000..0101e9067 --- /dev/null +++ b/.github/actions/sdk-test-testdatagen/action.yml @@ -0,0 +1,33 @@ +name: sdk-test-testdatagen +author: hpoeche + +description: Fetches generated example files from the aas-specs-metamodel repository and tries to deserialize them + +inputs: + AAS_SPECS_RELEASE_TAG: + required: true + description: A tag on the aas-specs-metamodel repository which defines the version of metamodel to test + +runs: + using: 'composite' + steps: + - name: Clone aas-specs-metamodel repository + shell: bash + env: + AAS_SPECS_RELEASE_TAG: ${{ inputs.AAS_SPECS_RELEASE_TAG }} + run: | + git clone https://github.com/admin-shell-io/aas-specs-metamodel.git + cd aas-specs-metamodel + git checkout $AAS_SPECS_RELEASE_TAG + + - name: Run tests on example files + shell: bash + run: | + python sdk/test/testdatagen/example_tests.py --examples aas-specs-metamodel/schemas --output testdata-output + + - name: Upload test output + if: ${{ always() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a #v7.0.1 + with: + path: testdata-output + name: sdk_test_testdatagen_results \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e71f1b5eb..40539c42a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -339,3 +339,19 @@ jobs: run: | docker stop basyx-python-server && docker rm basyx-python-server + tmp-sdk-testdatagen: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ env.X_PYTHON_MIN_VERSION }} + uses: actions/setup-python@v5 + with: + python-version: ${{ env.X_PYTHON_MIN_VERSION }} + - name: Install Python dependencies + working-directory: ./sdk + run: | + python -m pip install --upgrade pip + python -m pip install .[dev] + - uses: ./.github/actions/sdk-test-testdatagen + with: + AAS_SPECS_RELEASE_TAG: v3.1.2 \ No newline at end of file From 25eaa687d3cf1be8dd0ef33e7245f61420bbe002 Mon Sep 17 00:00:00 2001 From: hpoeche Date: Sun, 2 Aug 2026 19:03:26 +0200 Subject: [PATCH 05/14] Add output summary For usage in Github CI, the test script now outputs a markdown file comprehending the test results. For each format (JSON, XML) the number of passed and failed tests are shown, as well as a tabular overview on the amount of failed tests per error message. The summary also contains the sha of the commit of the `basyx-python-sdk` repo that the test ran on and the tag of the `aas-specs-metamodel` repository that provided the example files. --- .../actions/sdk-test-testdatagen/action.yml | 2 +- sdk/test/testdatagen/example_tests.py | 123 +++++++++++++++--- 2 files changed, 104 insertions(+), 21 deletions(-) diff --git a/.github/actions/sdk-test-testdatagen/action.yml b/.github/actions/sdk-test-testdatagen/action.yml index 0101e9067..8f5d91e0f 100644 --- a/.github/actions/sdk-test-testdatagen/action.yml +++ b/.github/actions/sdk-test-testdatagen/action.yml @@ -23,7 +23,7 @@ runs: - name: Run tests on example files shell: bash run: | - python sdk/test/testdatagen/example_tests.py --examples aas-specs-metamodel/schemas --output testdata-output + python sdk/test/testdatagen/example_tests.py --examples aas-specs-metamodel/schemas --output testdata-output --summary $GITHUB_STEP_SUMMARY - name: Upload test output if: ${{ always() }} diff --git a/sdk/test/testdatagen/example_tests.py b/sdk/test/testdatagen/example_tests.py index b13ee166d..cd0c07a77 100644 --- a/sdk/test/testdatagen/example_tests.py +++ b/sdk/test/testdatagen/example_tests.py @@ -1,8 +1,11 @@ import argparse +import enum import logging import re +import subprocess import sys import traceback +from collections import Counter from pathlib import Path from typing import Optional @@ -11,6 +14,13 @@ logger = logging.getLogger(__name__) +class TestResult(enum.Enum): + SUCCESS = "Success :white_check_mark:" + NOT_IMPLEMENTED = "Not Implemented :warning:" + FAILED = "Failed :x:" + UNEXPECTED_ERROR = "Unexpected Error :exclamation:" + def __format__(self, format_spec): + return f"{self.value}" def sanitize_name(name: str, max_len: int = 120) -> str: pattern = r'[<>:"/\\|?*\x00-\x1f\s\'{}\(\)\[\]]' @@ -18,8 +28,6 @@ def sanitize_name(name: str, max_len: int = 120) -> str: name = re.sub(r'_+', '_', name) return name.strip('_')[:max_len] - - def write_error_report(file_type, example_file, rel_path, error_dir, tb): error_dir.mkdir(parents=True, exist_ok=True) # Use the full relative path (not just the stem) to avoid collisions: generated example sets reuse @@ -31,12 +39,14 @@ def write_error_report(file_type, example_file, rel_path, error_dir, tb): error_file.write_text(error_output, encoding="utf-8") -def run_example_test(file_type, example_file, read_fn, base_path=None, output_dir=None, is_xml=False): +def run_example_test( + file_type, example_file, read_fn, base_path=None, output_dir=None, is_xml=False +) -> tuple[TestResult, str]: rel_path = example_file.relative_to(base_path) if base_path else example_file try: read_fn(example_file) - return True + return TestResult.SUCCESS, "" except (KeyError, TypeError, ValueError, AASConstraintViolation) as ex: tb = traceback.format_exc() error_msg = extract_error_message(ex) if is_xml else str(ex).split(">>>")[0].strip() @@ -45,7 +55,7 @@ def run_example_test(file_type, example_file, read_fn, base_path=None, output_di if output_dir: error_dir = output_dir / file_type / sanitize_name(error_msg) write_error_report(file_type, example_file, rel_path, error_dir, tb) - return False + return TestResult.FAILED, error_msg except NotImplementedError as ex: tb = traceback.format_exc() @@ -55,7 +65,7 @@ def run_example_test(file_type, example_file, read_fn, base_path=None, output_di if output_dir: error_dir = output_dir / file_type / "NotImplementedError" / sanitize_name(error_msg) write_error_report(file_type, example_file, rel_path, error_dir, tb) - return True + return TestResult.NOT_IMPLEMENTED, error_msg except Exception as ex: # Catch-all so an exception type not (yet) anticipated by the except clauses above (e.g. thrown by a @@ -74,7 +84,7 @@ def run_example_test(file_type, example_file, read_fn, base_path=None, output_di / sanitize_name(error_msg) ) write_error_report(file_type, example_file, rel_path, error_dir, tb) - return False + return TestResult.UNEXPECTED_ERROR, error_msg def extract_error_message(ex): message = str(ex) @@ -85,7 +95,9 @@ def extract_error_message(ex): return re.sub(r'on line [0-9]+', "", message.split("->")[-1]).strip() -def test_json_example(example_file: Path, base_path: Optional[Path] = None, output_dir: Optional[Path] = None) -> bool: +def test_json_example( + example_file: Path, base_path: Optional[Path] = None, output_dir: Optional[Path] = None +) -> tuple[TestResult, str]: """ Attempts to read and deserialize an example file. Reports the status and saves information to output path. @@ -103,7 +115,9 @@ def test_json_example(example_file: Path, base_path: Optional[Path] = None, outp ) -def test_xml_example(example_file: Path, base_path: Optional[Path] = None, output_dir: Optional[Path] = None) -> bool: +def test_xml_example( + example_file: Path, base_path: Optional[Path] = None, output_dir: Optional[Path] = None +) -> tuple[TestResult, str]: """ Attempts to read and deserialize an XML example file. Reports the status and saves information to output path. @@ -121,10 +135,72 @@ def test_xml_example(example_file: Path, base_path: Optional[Path] = None, outpu is_xml=True, ) +def summarize_output( + test_results: dict[str, Counter[tuple[TestResult, str]]], example_path: Path, output_file: Path +) -> None: + """Builds a Markdown summary of the failures recorded in `test_results`.""" -def main(example_path_str: str, output_path_str: Optional[str]) -> None: + # Get commit hash of sdk + try: + sdk_commit = subprocess.run( + ["git", "rev-parse", "--short", "HEAD"], + cwd=Path(__file__).parent, capture_output=True, text=True, check=True + ).stdout.strip() + except (subprocess.CalledProcessError, FileNotFoundError): + sdk_commit = "unknown" + + # Get version of example files (if in git repo) + try: + metamodel_origin = subprocess.run( + ["git", "remote", "get-url", "origin"], cwd=example_path, capture_output=True, text=True, check=True + ).stdout.strip() + + if "aas-specs-metamodel" not in metamodel_origin: + raise ValueError("Exaple files do not stem from aas-specs-metamodel repository") + metamodel_tag = subprocess.run( + ["git", "describe", "--tags", "--exact-match"], + cwd=example_path, capture_output=True, text=True, check=True + ).stdout.strip() + except (subprocess.CalledProcessError, FileNotFoundError, ValueError): + metamodel_tag = "unknown" + + sections = [f"## AAS testdatagen results (metamodel: `{metamodel_tag}`)", + f"Summary of the results from testing the SDK implementation (`{sdk_commit}`) against the example files" + f"from `aas-specs-metamodel` (version: `{metamodel_tag}`). For detailed information on each failed test" + f"consider the artifact uploaded with this job."] + + for format_name in test_results.keys(): + format_results = test_results[format_name] + sections.append(f"### {format_name}") + + totals: dict[TestResult, int] = {} + for category, err_msg in format_results.keys(): + totals[category] = totals.get(category, 0) + format_results[category, err_msg] + sections.append(" · ".join(f"{category} : {count}" for category, count in totals.items())) + + table_lines = [ + "
", + f"Error groups ({sum(totals.values())})", + "", + "| Category | Group | Count |", + "|---|---|---|", + ] + table_lines += [f"| {category} | {err_msg} | {count} |" + for (category, err_msg), count in format_results.most_common()] + table_lines += ["", "
"] + sections.append("\n".join(table_lines)) + + output_file.write_text("\n\n".join(sections)) + +def main(example_path_str: str, output_path_str: Optional[str], summary_path_str: Optional[str]) -> None: example_path = Path(example_path_str) output_path = Path(output_path_str) if output_path_str else None + summary_path = Path(summary_path_str) if summary_path_str else None + + if not example_path.exists(): + logger.error(f"Provided path for examples does not exist: {example_path}") + sys.exit(1) + if output_path: output_path.mkdir(parents=True, exist_ok=True) @@ -133,23 +209,29 @@ def main(example_path_str: str, output_path_str: Optional[str]) -> None: ("XML", example_path / "xml" / "examples" / "generated", "*.xml", test_xml_example), ] + test_results: dict[str, Counter[tuple[TestResult, str]]] = dict() total_failed = 0 for name, directory, glob_pattern, test_fn in test_configs: - total = 0 - failed = 0 + format_results: Counter[tuple[TestResult, str]] = Counter() for test_file in sorted(directory.glob(f"**/{glob_pattern}")): - total += 1 - if not test_fn(test_file, directory, output_path): - failed += 1 - + test_result = test_fn(test_file, directory, output_path) + format_results[test_result] += 1 + + test_results[name] = format_results + failed = sum((format_results[result, err_msg] + for result, err_msg in format_results.keys() + if result not in (TestResult.SUCCESS, TestResult.NOT_IMPLEMENTED) + )) + total_failed += failed + total = sum(format_results.values()) if failed == 0: logger.info(f"No {name} tests out of {total:d} failed") else: logger.error(f"Failed {failed:d}/{total:d} {name.lower()} tests") - total_failed += failed - - sys.exit(total_failed) + if summary_path is not None: + summarize_output(test_results, example_path, summary_path) + sys.exit(1 if total_failed > 0 else 0) if __name__ == '__main__': @@ -157,6 +239,7 @@ def main(example_path_str: str, output_path_str: Optional[str]) -> None: parser = argparse.ArgumentParser() parser.add_argument("--examples", required=True, help="path to examples directory") parser.add_argument("--output", required=False, help="path to output directory") + parser.add_argument("--summary", required=False, help="file to write summary to") args = parser.parse_args() - main(args.examples, args.output) \ No newline at end of file + main(args.examples, args.output, args.summary) From 8c5d293e5d0d7c731bb821fe31cee7e1d2870cf7 Mon Sep 17 00:00:00 2001 From: hpoeche Date: Sun, 2 Aug 2026 19:21:28 +0200 Subject: [PATCH 06/14] Fix: Do not include SUCESS into summary tables --- sdk/test/testdatagen/example_tests.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/sdk/test/testdatagen/example_tests.py b/sdk/test/testdatagen/example_tests.py index cd0c07a77..85624bea7 100644 --- a/sdk/test/testdatagen/example_tests.py +++ b/sdk/test/testdatagen/example_tests.py @@ -178,15 +178,18 @@ def summarize_output( totals[category] = totals.get(category, 0) + format_results[category, err_msg] sections.append(" · ".join(f"{category} : {count}" for category, count in totals.items())) + total_failed = sum((value for category, value in totals.items() if category != TestResult.SUCCESS)) table_lines = [ "
", - f"Error groups ({sum(totals.values())})", + f"Error groups ({total_failed})", "", "| Category | Group | Count |", "|---|---|---|", ] table_lines += [f"| {category} | {err_msg} | {count} |" - for (category, err_msg), count in format_results.most_common()] + for (category, err_msg), count in format_results.most_common() + if category != TestResult.SUCCESS + ] table_lines += ["", "
"] sections.append("\n".join(table_lines)) From 88a7833675d503afd085ed6697da73c8b087bba7 Mon Sep 17 00:00:00 2001 From: hpoeche Date: Thu, 6 Aug 2026 16:07:53 +0200 Subject: [PATCH 07/14] checkout aas-specs-metamodel via actions/checkout --- .github/actions/sdk-test-testdatagen/action.yml | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/.github/actions/sdk-test-testdatagen/action.yml b/.github/actions/sdk-test-testdatagen/action.yml index 8f5d91e0f..2c27573a0 100644 --- a/.github/actions/sdk-test-testdatagen/action.yml +++ b/.github/actions/sdk-test-testdatagen/action.yml @@ -11,14 +11,12 @@ inputs: runs: using: 'composite' steps: - - name: Clone aas-specs-metamodel repository - shell: bash - env: - AAS_SPECS_RELEASE_TAG: ${{ inputs.AAS_SPECS_RELEASE_TAG }} - run: | - git clone https://github.com/admin-shell-io/aas-specs-metamodel.git - cd aas-specs-metamodel - git checkout $AAS_SPECS_RELEASE_TAG + - name: Checkout aas-specs-metamodel repository + uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 #v5.1.0 + with: + repository: 'admin-shell-io/aas-specs-metamodel' + ref: ${{ inputs.AAS_SPECS_RELEASE_TAG }} + path: 'aas-specs-metamodel' - name: Run tests on example files shell: bash From d2bd220b8a9ae727ce6e4612569dfe1b97a4d390 Mon Sep 17 00:00:00 2001 From: hpoeche Date: Mon, 10 Aug 2026 10:36:25 +0200 Subject: [PATCH 08/14] Move testing script to `/etc/` and align naming --- .github/actions/sdk-test-testdatagen/action.yml | 9 ++++++--- {sdk/test/testdatagen => etc/sdk-testdatagen}/Readme.md | 0 .../sdk-testdatagen/deserialization_tests.py | 0 3 files changed, 6 insertions(+), 3 deletions(-) rename {sdk/test/testdatagen => etc/sdk-testdatagen}/Readme.md (100%) rename sdk/test/testdatagen/example_tests.py => etc/sdk-testdatagen/deserialization_tests.py (100%) diff --git a/.github/actions/sdk-test-testdatagen/action.yml b/.github/actions/sdk-test-testdatagen/action.yml index 2c27573a0..67cf0f09f 100644 --- a/.github/actions/sdk-test-testdatagen/action.yml +++ b/.github/actions/sdk-test-testdatagen/action.yml @@ -21,11 +21,14 @@ runs: - name: Run tests on example files shell: bash run: | - python sdk/test/testdatagen/example_tests.py --examples aas-specs-metamodel/schemas --output testdata-output --summary $GITHUB_STEP_SUMMARY + python etc/sdk-testdatagen/deserialization_tests.py \ + --examples aas-specs-metamodel/schemas \ + --output sdk-testdatagen-output \ + --summary $GITHUB_STEP_SUMMARY - name: Upload test output if: ${{ always() }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a #v7.0.1 with: - path: testdata-output - name: sdk_test_testdatagen_results \ No newline at end of file + path: sdk-testdatagen-output + name: sdk_testdatagen_results \ No newline at end of file diff --git a/sdk/test/testdatagen/Readme.md b/etc/sdk-testdatagen/Readme.md similarity index 100% rename from sdk/test/testdatagen/Readme.md rename to etc/sdk-testdatagen/Readme.md diff --git a/sdk/test/testdatagen/example_tests.py b/etc/sdk-testdatagen/deserialization_tests.py similarity index 100% rename from sdk/test/testdatagen/example_tests.py rename to etc/sdk-testdatagen/deserialization_tests.py From c47185f32a6b197361d0f14f5a37ec821000d645 Mon Sep 17 00:00:00 2001 From: hpoeche Date: Mon, 10 Aug 2026 11:22:03 +0200 Subject: [PATCH 09/14] Update `sdk-testdatagen` Readme.md --- etc/sdk-testdatagen/Readme.md | 39 ++++++++++++++++++++++++++++++----- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/etc/sdk-testdatagen/Readme.md b/etc/sdk-testdatagen/Readme.md index 827ffcbe9..a22a9e48c 100644 --- a/etc/sdk-testdatagen/Readme.md +++ b/etc/sdk-testdatagen/Readme.md @@ -1,16 +1,45 @@ ## Fuzzy Testing with `aas-core-testdatagen` This folder contains a script to test the implementation of the `basyx-python-sdk` against the example files -generated by the [`aas-core-works/aas-core-testdatagen`](https://github.com/aas-core-works/aas-core-testdatagen) project. +generated by the [`aas-core-works/aas-core-testdatagen`](https://github.com/aas-core-works/aas-core-testdatagen) project. In detail, the test script attempts to deserialize +each example file, keeping track of all occurring errors and generates a summary, showing error frequencies. ### Prerequisites The script requires the example files to exist in the specified path in the same structure as they are shipped in the -[`aas-specs-metamodel`](https://github.com/admin-shell-io/aas-specs-metamodel) repository. Specifically, if -`$EXAMPLES` is the path passed to the script the following directories are scanned for `.json` and `.xml` files respectively: +[`aas-specs-metamodel`](https://github.com/admin-shell-io/aas-specs-metamodel) repository. Specifically, if `{examples}` is the path passed to the script the following +directories are scanned for `.json` and `.xml` files respectively: ``` -$EXAMPLES/json/examples/generated -$EXAMPLES/xml/examples/generated +{examples}/json/examples/generated +{examples}/xml/examples/generated ``` Additionally, the `basyx-python-sdk` must be installed via `pip install sdk/.`. +### Running Tests +Call the Python script with `python etc/sdk-testdatagen/deserialization_tests.py` with the following options: + +| Argument | Required | Description | +|--------------|----------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `--examples` | Yes | Path to the directory containing the example files as described in [Prerequisites](#prerequisites) (usually this is the `/schemas` directory in a cloned `aas-specs-metamodel` respository.) | +| `--output` | No | Directory path where error reports are written | +| `--summary` | No | File path for generated markdown summary (useful in CI as Github step summary) | + +### Test Results +The script distinguishes 4 different results of a test. For all non-succeeding tests, an output report is written to the +`--output` directory containing the content of the example file that failed, as well as the resulting stack trace. +Files are sorted into subdirectories by their exception naming (e.g. `Value_is_not_a_valid_XSD_date_string`) and named +after the subpath in the `--examples` directory (e.g. `Extension_OverValueExamples_Date_fuzzed_05.json.txt`) + + - `SUCCESS`: The deserialization threw no exception + - `ERROR`: During deserialization a `KeyError`, `ValueError` or `TypeError` occurred. Indicates that the implementation + is not accepting a schema compliant example. + - `NOT IMPLEMENTED`: During deserialization a `NotImplementedError` was thrown. This indicates a well known deviation + of implementation and standard. Output reports are grouped in a `NotImplementedError` subdir. + - `UNEXPECTED`: During deserialization an unexpected Exception (e.g. `IOError`) occurred. This is no implementation + issue but should not crash the script. Therefore, these reports are grouped into an `UnexpectedError` subdir. + +## Usage in GitHub CI +For convenient usage in the GitHub CI a composite action is provided in `.github/actions/sdk-test-testdatagen/action.yml`. +This action pulls the version of the `aas-specs-metamodel` specified in the `AAS_SPECS_RELEASE_TAG` input variable and +runs the deserialization tests. The Markdown summary is output as job summary on GitHub and the output reports are +uploaded as artifact. \ No newline at end of file From 7d441fca68ced75b847a3679073d1e6008546266 Mon Sep 17 00:00:00 2001 From: hpoeche Date: Mon, 10 Aug 2026 11:33:49 +0200 Subject: [PATCH 10/14] CI: Integrate `sdk-testdatagen` into main.yml --- .github/workflows/main.yml | 21 +++++++++++++++++++++ .github/workflows/pr.yml | 17 ----------------- 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c3f4f537d..2dcfd9a24 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -12,6 +12,8 @@ concurrency: env: X_API_VERSION: "v3.1" + X_PYTHON_MIN_VERSION: "3.10" + X_AAS_SPECS_RELEASE_TAG: "v3.1.2" jobs: server-docker-arm: @@ -48,3 +50,22 @@ jobs: - name: Stop and remove the container run: | docker stop basyx-python-${{ matrix.profile }} && docker rm basyx-python-${{ matrix.profile }} + + sdk-testdatagen: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 #v5.1.0 + - name: Set up Python + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v7 + with: + python-version: ${{ env.X_PYTHON_MIN_VERSION }} + cache: "pip" + cache-dependency-path: "**/pyproject.toml" + - name: Install Python dependencies + working-directory: ./sdk + run: | + python -m pip install --upgrade pip + python -m pip install .[dev] + - uses: ./.github/actions/sdk-test-testdatagen + with: + AAS_SPECS_RELEASE_TAG: ${{ env.X_AAS_SPECS_RELEASE_TAG }} diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 9d0ee1b29..ce3008e4d 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -405,20 +405,3 @@ jobs: - name: Stop and remove the container run: | docker stop basyx-python-${{ matrix.profile }} && docker rm basyx-python-${{ matrix.profile }} - - tmp-sdk-testdatagen: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Set up Python ${{ env.X_PYTHON_MIN_VERSION }} - uses: actions/setup-python@v5 - with: - python-version: ${{ env.X_PYTHON_MIN_VERSION }} - - name: Install Python dependencies - working-directory: ./sdk - run: | - python -m pip install --upgrade pip - python -m pip install .[dev] - - uses: ./.github/actions/sdk-test-testdatagen - with: - AAS_SPECS_RELEASE_TAG: v3.1.2 \ No newline at end of file From c5309d26c88e4cb6fa60b06142e7efc65cf57587 Mon Sep 17 00:00:00 2001 From: hpoeche Date: Fri, 18 Sep 2026 14:53:28 +0200 Subject: [PATCH 11/14] CI: Adapt sdk-testdatagen run to singe-point-of-truth --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4091cc754..14a226904 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -52,6 +52,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 #v5.1.0 + - uses: ./.github/actions/load-versions - name: Set up Python uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v7 with: From 16d29ef04f8f3196031fc74b6d54a6578f0bfe81 Mon Sep 17 00:00:00 2001 From: s-heppner Date: Tue, 6 Oct 2026 10:28:17 +0200 Subject: [PATCH 12/14] sdk-testdatagen: Fix summary count, docs and lint Previously, the "Error groups" count in the Markdown summary included `NOT_IMPLEMENTED` results, although these do not make the script fail. The `Readme.md` used result names that did not match the `TestResult` enum, and the script was not checked by any linter. With this change: - A `FAILING_RESULTS` constant defines the failing results for both the summary count and the exit code. The summary now shows the failed and not implemented counts separately. - The `Readme.md` uses the enum names, lists `AASConstraintViolation` and documents the exit code. - The `helper-scripts-static-analysis` job now runs `ruff check` on all of `etc`, which includes `etc/sdk-testdatagen`. - The script has a module docstring, and the helper functions have type hints and docstrings. - Smaller fixes: missing spaces and a typo in the summary text, the indentation in `main.yml`, missing newlines at end of file and the `Readme.md` heading. --- .../actions/sdk-test-testdatagen/action.yml | 2 +- .github/workflows/main.yml | 32 ++++---- .github/workflows/pr.yml | 4 +- etc/sdk-testdatagen/Readme.md | 26 ++++--- etc/sdk-testdatagen/deserialization_tests.py | 75 +++++++++++++++---- 5 files changed, 96 insertions(+), 43 deletions(-) diff --git a/.github/actions/sdk-test-testdatagen/action.yml b/.github/actions/sdk-test-testdatagen/action.yml index 67cf0f09f..29252bb4b 100644 --- a/.github/actions/sdk-test-testdatagen/action.yml +++ b/.github/actions/sdk-test-testdatagen/action.yml @@ -31,4 +31,4 @@ runs: uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a #v7.0.1 with: path: sdk-testdatagen-output - name: sdk_testdatagen_results \ No newline at end of file + name: sdk_testdatagen_results diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 14a226904..da4a16e64 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -51,19 +51,19 @@ jobs: sdk-testdatagen: runs-on: ubuntu-latest steps: - - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 #v5.1.0 - - uses: ./.github/actions/load-versions - - name: Set up Python - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v7 - with: - python-version: ${{ env.X_PYTHON_MIN_VERSION }} - cache: "pip" - cache-dependency-path: "**/pyproject.toml" - - name: Install Python dependencies - working-directory: ./sdk - run: | - python -m pip install --upgrade pip - python -m pip install .[dev] - - uses: ./.github/actions/sdk-test-testdatagen - with: - AAS_SPECS_RELEASE_TAG: ${{ env.X_AAS_SPECS_RELEASE_TAG }} + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 #v5.1.0 + - uses: ./.github/actions/load-versions + - name: Set up Python + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v7 + with: + python-version: ${{ env.X_PYTHON_MIN_VERSION }} + cache: "pip" + cache-dependency-path: "**/pyproject.toml" + - name: Install Python dependencies + working-directory: ./sdk + run: | + python -m pip install --upgrade pip + python -m pip install .[dev] + - uses: ./.github/actions/sdk-test-testdatagen + with: + AAS_SPECS_RELEASE_TAG: ${{ env.X_AAS_SPECS_RELEASE_TAG }} diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index bd0d5d772..c7eaf3f1d 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -14,11 +14,11 @@ concurrency: jobs: helper-scripts-static-analysis: - #This job runs static code analysis with ruff on the helper scripts in ./etc/scripts + #This job runs static code analysis with ruff on the helper scripts in ./etc runs-on: ubuntu-latest defaults: run: - working-directory: ./etc/scripts + working-directory: ./etc steps: - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 #v5.1.0 - uses: ./.github/actions/load-versions diff --git a/etc/sdk-testdatagen/Readme.md b/etc/sdk-testdatagen/Readme.md index a22a9e48c..b75fe41a1 100644 --- a/etc/sdk-testdatagen/Readme.md +++ b/etc/sdk-testdatagen/Readme.md @@ -1,4 +1,4 @@ -## Fuzzy Testing with `aas-core-testdatagen` +## Fuzz Testing with `aas-core-testdatagen` This folder contains a script to test the implementation of the `basyx-python-sdk` against the example files generated by the [`aas-core-works/aas-core-testdatagen`](https://github.com/aas-core-works/aas-core-testdatagen) project. In detail, the test script attempts to deserialize each example file, keeping track of all occurring errors and generates a summary, showing error frequencies. @@ -27,19 +27,23 @@ Call the Python script with `python etc/sdk-testdatagen/deserialization_tests.py ### Test Results The script distinguishes 4 different results of a test. For all non-succeeding tests, an output report is written to the `--output` directory containing the content of the example file that failed, as well as the resulting stack trace. -Files are sorted into subdirectories by their exception naming (e.g. `Value_is_not_a_valid_XSD_date_string`) and named -after the subpath in the `--examples` directory (e.g. `Extension_OverValueExamples_Date_fuzzed_05.json.txt`) - - - `SUCCESS`: The deserialization threw no exception - - `ERROR`: During deserialization a `KeyError`, `ValueError` or `TypeError` occurred. Indicates that the implementation - is not accepting a schema compliant example. - - `NOT IMPLEMENTED`: During deserialization a `NotImplementedError` was thrown. This indicates a well known deviation +Reports are sorted into subdirectories by format (`json`, `xml`) and by their error message +(e.g. `Value_is_not_a_valid_XSD_date_string`), and named after the subpath of the example file in the `generated` +directory (e.g. `Extension_OverValueExamples_Date_fuzzed_05.json.txt`). + + - `SUCCESS`: The deserialization threw no exception. + - `FAILED`: During deserialization a `KeyError`, `ValueError`, `TypeError` or `AASConstraintViolation` occurred. + Indicates that the implementation is not accepting a schema compliant example. + - `NOT_IMPLEMENTED`: During deserialization a `NotImplementedError` was thrown. This indicates a well known deviation of implementation and standard. Output reports are grouped in a `NotImplementedError` subdir. - - `UNEXPECTED`: During deserialization an unexpected Exception (e.g. `IOError`) occurred. This is no implementation - issue but should not crash the script. Therefore, these reports are grouped into an `UnexpectedError` subdir. + - `UNEXPECTED_ERROR`: During deserialization any other exception occurred. It is caught so that the remaining examples + are still tested. Output reports are grouped by exception type in an `UnexpectedError` subdir. + +The script exits with code `1` if at least one test has the result `FAILED` or `UNEXPECTED_ERROR`, otherwise with `0`. +`NOT_IMPLEMENTED` results do not make the script fail. ## Usage in GitHub CI For convenient usage in the GitHub CI a composite action is provided in `.github/actions/sdk-test-testdatagen/action.yml`. This action pulls the version of the `aas-specs-metamodel` specified in the `AAS_SPECS_RELEASE_TAG` input variable and runs the deserialization tests. The Markdown summary is output as job summary on GitHub and the output reports are -uploaded as artifact. \ No newline at end of file +uploaded as artifact. diff --git a/etc/sdk-testdatagen/deserialization_tests.py b/etc/sdk-testdatagen/deserialization_tests.py index 85624bea7..33464909b 100644 --- a/etc/sdk-testdatagen/deserialization_tests.py +++ b/etc/sdk-testdatagen/deserialization_tests.py @@ -1,3 +1,15 @@ +""" +This script tests the deserialization of the BaSyx Python SDK against the example files that +`aas-core-testdatagen` generates. The `aas-specs-metamodel` repository supplies these example files. + +The script reads each JSON and XML example file with the SDK adapters. It puts the result of each test into one of +the categories in `TestResult`. For each test that does not succeed, the script can write a report that contains the +example file and the stack trace. It can also write a Markdown summary, which the CI shows as the GitHub job summary. + +The script exits with code 1 if at least one test has a result in `FAILING_RESULTS`. + +For the usage and the arguments, refer to the `Readme.md` in this directory. +""" import argparse import enum import logging @@ -7,7 +19,7 @@ import traceback from collections import Counter from pathlib import Path -from typing import Optional +from typing import Callable, Optional from basyx.aas import adapter from basyx.aas.model import AASConstraintViolation @@ -22,13 +34,25 @@ class TestResult(enum.Enum): def __format__(self, format_spec): return f"{self.value}" +# Results that count as failures, both in the summary and for the exit code of the script +FAILING_RESULTS = (TestResult.FAILED, TestResult.UNEXPECTED_ERROR) + def sanitize_name(name: str, max_len: int = 120) -> str: pattern = r'[<>:"/\\|?*\x00-\x1f\s\'{}\(\)\[\]]' name = re.sub(pattern, '_', name) name = re.sub(r'_+', '_', name) return name.strip('_')[:max_len] -def write_error_report(file_type, example_file, rel_path, error_dir, tb): +def write_error_report(file_type: str, example_file: Path, rel_path: Path, error_dir: Path, tb: str) -> None: + """ + Writes a report with the content of the example file and the stack trace of the error. + + :param file_type: Format of the example file (`json` or `xml`) + :param example_file: File which contains the example that failed + :param rel_path: Path of the example file relative to the examples directory, used to name the report + :param error_dir: Directory to write the report to + :param tb: Formatted stack trace of the error + """ error_dir.mkdir(parents=True, exist_ok=True) # Use the full relative path (not just the stem) to avoid collisions: generated example sets reuse # the same filenames across many different subdirectories. @@ -40,8 +64,25 @@ def write_error_report(file_type, example_file, rel_path, error_dir, tb): def run_example_test( - file_type, example_file, read_fn, base_path=None, output_dir=None, is_xml=False + file_type: str, + example_file: Path, + read_fn: Callable[[Path], object], + base_path: Optional[Path] = None, + output_dir: Optional[Path] = None, + is_xml: bool = False, ) -> tuple[TestResult, str]: + """ + Deserializes an example file with `read_fn` and categorizes the outcome. Writes an error report to `output_dir` + for each test that does not succeed. + + :param file_type: Format of the example file (`json` or `xml`), used for logging and the report directory + :param example_file: File which contains the example that should be deserialized + :param read_fn: Function that deserializes the example file + :param base_path: Base directory to compute relative paths + :param output_dir: Path to store the failed outputs + :param is_xml: If True, the error message is extracted from the chain of causes of the exception + :return: Result of the test and the error message (empty on success) + """ rel_path = example_file.relative_to(base_path) if base_path else example_file try: @@ -86,7 +127,14 @@ def run_example_test( write_error_report(file_type, example_file, rel_path, error_dir, tb) return TestResult.UNEXPECTED_ERROR, error_msg -def extract_error_message(ex): +def extract_error_message(ex: BaseException) -> str: + """ + Extracts the message of the innermost cause of an exception, without line numbers, so that errors of different + example files can be grouped. + + :param ex: Exception to extract the message from + :return: Message of the innermost cause + """ message = str(ex) cause = ex.__cause__ while cause is not None: @@ -104,7 +152,7 @@ def test_json_example( :param example_file: File which contains the example that should be deserialized :param base_path: Base directory to compute relative paths :param output_dir: Path to store the failed outputs - :return: bool that indicates if the example was successfully deserialized + :return: Result of the test and the error message (empty on success) """ return run_example_test( file_type="json", @@ -124,7 +172,7 @@ def test_xml_example( :param example_file: File which contains the example that should be deserialized :param base_path: Base directory to compute relative paths :param output_dir: Path to store the failed outputs - :return: bool that indicates if the example was successfully deserialized + :return: Result of the test and the error message (empty on success) """ return run_example_test( file_type="xml", @@ -156,7 +204,7 @@ def summarize_output( ).stdout.strip() if "aas-specs-metamodel" not in metamodel_origin: - raise ValueError("Exaple files do not stem from aas-specs-metamodel repository") + raise ValueError("Example files do not stem from aas-specs-metamodel repository") metamodel_tag = subprocess.run( ["git", "describe", "--tags", "--exact-match"], cwd=example_path, capture_output=True, text=True, check=True @@ -165,9 +213,9 @@ def summarize_output( metamodel_tag = "unknown" sections = [f"## AAS testdatagen results (metamodel: `{metamodel_tag}`)", - f"Summary of the results from testing the SDK implementation (`{sdk_commit}`) against the example files" - f"from `aas-specs-metamodel` (version: `{metamodel_tag}`). For detailed information on each failed test" - f"consider the artifact uploaded with this job."] + f"Summary of the results from testing the SDK implementation (`{sdk_commit}`) against the example " + f"files from `aas-specs-metamodel` (version: `{metamodel_tag}`). For detailed information on each " + f"failed test consider the artifact uploaded with this job."] for format_name in test_results.keys(): format_results = test_results[format_name] @@ -178,10 +226,11 @@ def summarize_output( totals[category] = totals.get(category, 0) + format_results[category, err_msg] sections.append(" · ".join(f"{category} : {count}" for category, count in totals.items())) - total_failed = sum((value for category, value in totals.items() if category != TestResult.SUCCESS)) + total_failed = sum(value for category, value in totals.items() if category in FAILING_RESULTS) + total_not_implemented = totals.get(TestResult.NOT_IMPLEMENTED, 0) table_lines = [ "
", - f"Error groups ({total_failed})", + f"Error groups ({total_failed} failed, {total_not_implemented} not implemented)", "", "| Category | Group | Count |", "|---|---|---|", @@ -223,7 +272,7 @@ def main(example_path_str: str, output_path_str: Optional[str], summary_path_str test_results[name] = format_results failed = sum((format_results[result, err_msg] for result, err_msg in format_results.keys() - if result not in (TestResult.SUCCESS, TestResult.NOT_IMPLEMENTED) + if result in FAILING_RESULTS )) total_failed += failed total = sum(format_results.values()) From 76c31f847224a4b23f69b81cbc92ac641756b015 Mon Sep 17 00:00:00 2001 From: hpoeche Date: Thu, 22 Oct 2026 17:16:47 +0200 Subject: [PATCH 13/14] Add `KNOWN_DEVIATION` status These changes introduce a fifth test result status for errors that are known and tracked. Error groups can be marked as `KNOWN_DEVIATION` by adding entries to the `known_dviations.toml` file. The script reads this file and marks the groups, identified by the error message, as known deviations from the spec. Know deviations do not fail the script and leave the CI green. The script enforces each known deviation to come with a link to a GitHub issue and checks that no stale entries stay in the file. --- .../actions/sdk-test-testdatagen/action.yml | 1 + etc/sdk-testdatagen/Readme.md | 32 ++- etc/sdk-testdatagen/deserialization_tests.py | 247 +++++++++++++----- etc/sdk-testdatagen/known_deviations.toml | 16 ++ 4 files changed, 229 insertions(+), 67 deletions(-) create mode 100644 etc/sdk-testdatagen/known_deviations.toml diff --git a/.github/actions/sdk-test-testdatagen/action.yml b/.github/actions/sdk-test-testdatagen/action.yml index 29252bb4b..29f9896a5 100644 --- a/.github/actions/sdk-test-testdatagen/action.yml +++ b/.github/actions/sdk-test-testdatagen/action.yml @@ -24,6 +24,7 @@ runs: python etc/sdk-testdatagen/deserialization_tests.py \ --examples aas-specs-metamodel/schemas \ --output sdk-testdatagen-output \ + --known-deviations etc/sdk-testdatagen/known_deviations.toml \ --summary $GITHUB_STEP_SUMMARY - name: Upload test output diff --git a/etc/sdk-testdatagen/Readme.md b/etc/sdk-testdatagen/Readme.md index b75fe41a1..71f1cab2f 100644 --- a/etc/sdk-testdatagen/Readme.md +++ b/etc/sdk-testdatagen/Readme.md @@ -23,9 +23,10 @@ Call the Python script with `python etc/sdk-testdatagen/deserialization_tests.py | `--examples` | Yes | Path to the directory containing the example files as described in [Prerequisites](#prerequisites) (usually this is the `/schemas` directory in a cloned `aas-specs-metamodel` respository.) | | `--output` | No | Directory path where error reports are written | | `--summary` | No | File path for generated markdown summary (useful in CI as Github step summary) | +| `--known-deviations` | No | File path of the known deviations file (default: `known_deviations.toml` in this directory), see [Known Deviations](#known-deviations) | ### Test Results -The script distinguishes 4 different results of a test. For all non-succeeding tests, an output report is written to the +The script distinguishes 5 different results of a test. For all non-succeeding tests, an output report is written to the `--output` directory containing the content of the example file that failed, as well as the resulting stack trace. Reports are sorted into subdirectories by format (`json`, `xml`) and by their error message (e.g. `Value_is_not_a_valid_XSD_date_string`), and named after the subpath of the example file in the `generated` @@ -38,9 +39,34 @@ directory (e.g. `Extension_OverValueExamples_Date_fuzzed_05.json.txt`). of implementation and standard. Output reports are grouped in a `NotImplementedError` subdir. - `UNEXPECTED_ERROR`: During deserialization any other exception occurred. It is caught so that the remaining examples are still tested. Output reports are grouped by exception type in an `UnexpectedError` subdir. + - `KNOWN_DEVIATION`: The error group of the test is listed in the known deviations file. This overrides the results + `FAILED`, `NOT_IMPLEMENTED` and `UNEXPECTED_ERROR`. Output reports are grouped in a `KnownDeviation` subdir. -The script exits with code `1` if at least one test has the result `FAILED` or `UNEXPECTED_ERROR`, otherwise with `0`. -`NOT_IMPLEMENTED` results do not make the script fail. +The script exits with code `1` if at least one test has the result `FAILED` or `UNEXPECTED_ERROR`, if no example files +are found for a format, or if a known deviation is stale (see below). Otherwise, it exits with `0`. +`NOT_IMPLEMENTED` and `KNOWN_DEVIATION` results do not make the script fail. + +### Known Deviations +Error groups that are known deviations of the implementation from the standard are listed in +`known_deviations.toml`. Each entry corresponds to one row of the error groups table in the summary and is listed +separately per format: + +```toml +# Python's datetime does not support negative years +[[XML]] +group = "Negative dates are not supported: Python stdlib datetime requires year >= 1. Report at ..." +issue = "https://github.com/eclipse-basyx/basyx-python-sdk/issues/" +``` + + - `group`: The exact message of the error group, as shown in the "Group" column of the summary. Messages that contain + the fuzzed value need one entry per value. + - `issue`: A link to the issue that describes the deviation in detail. It is required. + +The reason for the deviation should be briefly stated in a comment above the entry. Other keys are not allowed, and +the script exits with code `1` before running any test if the file is invalid. + +A known deviation whose group does not occur in any test is *stale*. Stale known deviations are listed in the summary +and make the script fail, so that the file is updated when a deviation is resolved. ## Usage in GitHub CI For convenient usage in the GitHub CI a composite action is provided in `.github/actions/sdk-test-testdatagen/action.yml`. diff --git a/etc/sdk-testdatagen/deserialization_tests.py b/etc/sdk-testdatagen/deserialization_tests.py index 33464909b..9bde1982e 100644 --- a/etc/sdk-testdatagen/deserialization_tests.py +++ b/etc/sdk-testdatagen/deserialization_tests.py @@ -1,12 +1,20 @@ +# Copyright (c) 2026 the Eclipse BaSyx Authors +# +# This program and the accompanying materials are made available under the terms of the MIT License, available in +# the LICENSE file of this project. +# +# SPDX-License-Identifier: MIT """ This script tests the deserialization of the BaSyx Python SDK against the example files that `aas-core-testdatagen` generates. The `aas-specs-metamodel` repository supplies these example files. The script reads each JSON and XML example file with the SDK adapters. It puts the result of each test into one of -the categories in `TestResult`. For each test that does not succeed, the script can write a report that contains the +the categories in `TestResult`. Error groups listed in the known deviations file (`known_deviations.toml`) are +categorized as `KNOWN_DEVIATION`. For each test that does not succeed, the script can write a report that contains the example file and the stack trace. It can also write a Markdown summary, which the CI shows as the GitHub job summary. -The script exits with code 1 if at least one test has a result in `FAILING_RESULTS`. +The script exits with code 1 if at least one test has a result in `FAILING_RESULTS`, if no example files are found for a +format, or if a known deviation did not occur (stale known deviation). For the usage and the arguments, refer to the `Readme.md` in this directory. """ @@ -16,10 +24,11 @@ import re import subprocess import sys +import tomllib import traceback from collections import Counter from pathlib import Path -from typing import Callable, Optional +from typing import Callable, Optional, Sequence from basyx.aas import adapter from basyx.aas.model import AASConstraintViolation @@ -31,12 +40,53 @@ class TestResult(enum.Enum): NOT_IMPLEMENTED = "Not Implemented :warning:" FAILED = "Failed :x:" UNEXPECTED_ERROR = "Unexpected Error :exclamation:" + KNOWN_DEVIATION = "Known Deviation :information_source:" def __format__(self, format_spec): return f"{self.value}" # Results that count as failures, both in the summary and for the exit code of the script FAILING_RESULTS = (TestResult.FAILED, TestResult.UNEXPECTED_ERROR) +# Formats that are tested, as used for the tables in the known deviations file +FORMATS = ("JSON", "XML") +KNOWN_DEVIATIONS_FILE = Path(__file__).parent / "known_deviations.toml" +ISSUE_LINK_RE = re.compile(r"^https?://\S+$") + + +def load_known_deviations(path: Path) -> dict[str, dict[str, str]]: + """ + Reads and validates the known deviations file. + + :param path: Path of the TOML file that lists the known deviations + :return: Mapping from format to a mapping from error group to issue link + :raises ValueError: If the content of the file is invalid + """ + with path.open("rb") as f: + data = tomllib.load(f) + + unknown_formats = set(data) - set(FORMATS) + if unknown_formats: + raise ValueError(f"Unknown formats {sorted(unknown_formats)}, expected one of {list(FORMATS)}") + + known_deviations: dict[str, dict[str, str]] = {format_name: {} for format_name in FORMATS} + for format_name, entries in data.items(): + if not isinstance(entries, list): + raise ValueError(f"'{format_name}' must be an array of tables ([[{format_name}]])") + for index, entry in enumerate(entries): + location = f"[[{format_name}]] entry {index + 1}" + if not isinstance(entry, dict) or set(entry) != {"group", "issue"}: + raise ValueError(f"{location} must have exactly the keys 'group' and 'issue'") + group, issue = entry["group"], entry["issue"] + if not isinstance(group, str) or not group.strip(): + raise ValueError(f"{location}: 'group' must be a non-empty string") + if not isinstance(issue, str) or not ISSUE_LINK_RE.match(issue): + raise ValueError(f"{location}: 'issue' must be a link, got {issue!r}") + if group in known_deviations[format_name]: + raise ValueError(f"{location}: duplicate group {group!r}") + known_deviations[format_name][group] = issue + return known_deviations + + def sanitize_name(name: str, max_len: int = 120) -> str: pattern = r'[<>:"/\\|?*\x00-\x1f\s\'{}\(\)\[\]]' name = re.sub(pattern, '_', name) @@ -70,6 +120,7 @@ def run_example_test( base_path: Optional[Path] = None, output_dir: Optional[Path] = None, is_xml: bool = False, + known_deviations: Optional[dict[str, str]] = None, ) -> tuple[TestResult, str]: """ Deserializes an example file with `read_fn` and categorizes the outcome. Writes an error report to `output_dir` @@ -81,51 +132,45 @@ def run_example_test( :param base_path: Base directory to compute relative paths :param output_dir: Path to store the failed outputs :param is_xml: If True, the error message is extracted from the chain of causes of the exception + :param known_deviations: Mapping from error group to issue link. Errors of these groups are categorized as + `KNOWN_DEVIATION`, regardless of the type of the exception. :return: Result of the test and the error message (empty on success) """ rel_path = example_file.relative_to(base_path) if base_path else example_file - try: - read_fn(example_file) - return TestResult.SUCCESS, "" - except (KeyError, TypeError, ValueError, AASConstraintViolation) as ex: + def handle_error(ex: Exception, result: TestResult, report_subdirs: Sequence[str]) -> tuple[TestResult, str]: + # Must be called from within the except clause, so that the stack trace is still available tb = traceback.format_exc() error_msg = extract_error_message(ex) if is_xml else str(ex).split(">>>")[0].strip() - logger.error(f"[{file_type.upper()}] ERROR on {rel_path}: {error_msg}") - if output_dir: - error_dir = output_dir / file_type / sanitize_name(error_msg) - write_error_report(file_type, example_file, rel_path, error_dir, tb) - return TestResult.FAILED, error_msg - - except NotImplementedError as ex: - tb = traceback.format_exc() - error_msg = extract_error_message(ex) if is_xml else str(ex).split(">>>")[0].strip() - logger.warning(f"[{file_type.upper()}] NOT_IMPLEMENTED on {rel_path}: {error_msg}") + if known_deviations and error_msg in known_deviations: + result, report_subdirs = TestResult.KNOWN_DEVIATION, ("KnownDeviation",) + logger.info(f"[{file_type.upper()}] KNOWN_DEVIATION on {rel_path}: {error_msg} " + f"(see {known_deviations[error_msg]})") + elif result == TestResult.NOT_IMPLEMENTED: + logger.warning(f"[{file_type.upper()}] NOT_IMPLEMENTED on {rel_path}: {error_msg}") + elif result == TestResult.UNEXPECTED_ERROR: + logger.error(f"[{file_type.upper()}] UNEXPECTED {type(ex).__name__} on {rel_path}: {error_msg}") + else: + logger.error(f"[{file_type.upper()}] ERROR on {rel_path}: {error_msg}") if output_dir: - error_dir = output_dir / file_type / "NotImplementedError" / sanitize_name(error_msg) + error_dir = output_dir.joinpath(file_type, *report_subdirs, sanitize_name(error_msg)) write_error_report(file_type, example_file, rel_path, error_dir, tb) - return TestResult.NOT_IMPLEMENTED, error_msg + return result, error_msg + try: + read_fn(example_file) + return TestResult.SUCCESS, "" + except (KeyError, TypeError, ValueError, AASConstraintViolation) as ex: + return handle_error(ex, TestResult.FAILED, ()) + except NotImplementedError as ex: + return handle_error(ex, TestResult.NOT_IMPLEMENTED, ("NotImplementedError",)) except Exception as ex: # Catch-all so an exception type not (yet) anticipated by the except clauses above (e.g. thrown by a # future implementation change) fails just this one example instead of aborting the whole run. # Deliberately not `except BaseException`, so KeyboardInterrupt/SystemExit still propagate. - tb = traceback.format_exc() - error_msg = extract_error_message(ex) if is_xml else str(ex).split(">>>")[0].strip() - logger.error(f"[{file_type.upper()}] UNEXPECTED {type(ex).__name__} on {rel_path}: {error_msg}") - - if output_dir: - error_dir = ( - output_dir - / file_type - / "UnexpectedError" - / sanitize_name(type(ex).__name__) - / sanitize_name(error_msg) - ) - write_error_report(file_type, example_file, rel_path, error_dir, tb) - return TestResult.UNEXPECTED_ERROR, error_msg + return handle_error(ex, TestResult.UNEXPECTED_ERROR, ("UnexpectedError", sanitize_name(type(ex).__name__))) def extract_error_message(ex: BaseException) -> str: """ @@ -144,7 +189,10 @@ def extract_error_message(ex: BaseException) -> str: def test_json_example( - example_file: Path, base_path: Optional[Path] = None, output_dir: Optional[Path] = None + example_file: Path, + base_path: Optional[Path] = None, + output_dir: Optional[Path] = None, + known_deviations: Optional[dict[str, str]] = None, ) -> tuple[TestResult, str]: """ Attempts to read and deserialize an example file. Reports the status and saves information to output path. @@ -152,6 +200,7 @@ def test_json_example( :param example_file: File which contains the example that should be deserialized :param base_path: Base directory to compute relative paths :param output_dir: Path to store the failed outputs + :param known_deviations: Mapping from error group to issue link of the known deviations :return: Result of the test and the error message (empty on success) """ return run_example_test( @@ -160,11 +209,15 @@ def test_json_example( read_fn=lambda f: adapter.json.read_aas_json_file(f, failsafe=False), base_path=base_path, output_dir=output_dir, + known_deviations=known_deviations, ) def test_xml_example( - example_file: Path, base_path: Optional[Path] = None, output_dir: Optional[Path] = None + example_file: Path, + base_path: Optional[Path] = None, + output_dir: Optional[Path] = None, + known_deviations: Optional[dict[str, str]] = None, ) -> tuple[TestResult, str]: """ Attempts to read and deserialize an XML example file. Reports the status and saves information to output path. @@ -172,6 +225,7 @@ def test_xml_example( :param example_file: File which contains the example that should be deserialized :param base_path: Base directory to compute relative paths :param output_dir: Path to store the failed outputs + :param known_deviations: Mapping from error group to issue link of the known deviations :return: Result of the test and the error message (empty on success) """ return run_example_test( @@ -181,12 +235,25 @@ def test_xml_example( base_path=base_path, output_dir=output_dir, is_xml=True, + known_deviations=known_deviations, ) def summarize_output( - test_results: dict[str, Counter[tuple[TestResult, str]]], example_path: Path, output_file: Path + test_results: dict[str, Counter[tuple[TestResult, str]]], + known_deviations: dict[str, dict[str, str]], + stale_deviations: dict[str, dict[str, str]], + example_path: Path, + output_file: Path, ) -> None: - """Builds a Markdown summary of the failures recorded in `test_results`.""" + """ + Builds a Markdown summary of the failures recorded in `test_results`. + + :param test_results: Count of each pair of result and error group, per format + :param known_deviations: Mapping from error group to issue link of the known deviations, per format + :param stale_deviations: Known deviations that did not occur, per format + :param example_path: Path of the examples directory, used to determine the version of the example files + :param output_file: File to write the summary to + """ # Get commit hash of sdk try: @@ -219,40 +286,75 @@ def summarize_output( for format_name in test_results.keys(): format_results = test_results[format_name] + format_known_deviations = known_deviations[format_name] sections.append(f"### {format_name}") - - totals: dict[TestResult, int] = {} - for category, err_msg in format_results.keys(): - totals[category] = totals.get(category, 0) + format_results[category, err_msg] - sections.append(" · ".join(f"{category} : {count}" for category, count in totals.items())) - - total_failed = sum(value for category, value in totals.items() if category in FAILING_RESULTS) - total_not_implemented = totals.get(TestResult.NOT_IMPLEMENTED, 0) - table_lines = [ - "
", - f"Error groups ({total_failed} failed, {total_not_implemented} not implemented)", - "", - "| Category | Group | Count |", - "|---|---|---|", - ] - table_lines += [f"| {category} | {err_msg} | {count} |" - for (category, err_msg), count in format_results.most_common() - if category != TestResult.SUCCESS - ] - table_lines += ["", "
"] - sections.append("\n".join(table_lines)) + if not format_results: + sections.append(f":x: No {format_name} example files found") + else: + totals: dict[TestResult, int] = {} + for category, err_msg in format_results.keys(): + totals[category] = totals.get(category, 0) + format_results[category, err_msg] + sections.append(" · ".join(f"{category} : {count}" for category, count in totals.items())) + + total_failed = sum(value for category, value in totals.items() if category in FAILING_RESULTS) + total_not_implemented = totals.get(TestResult.NOT_IMPLEMENTED, 0) + total_known_deviations = totals.get(TestResult.KNOWN_DEVIATION, 0) + table_lines = [ + "
", + f"Error groups ({total_failed} failed, {total_not_implemented} not implemented, " + f"{total_known_deviations} known deviations)", + "", + "| Category | Group | Issue | Count |", + "|---|---|---|---|", + ] + table_lines += [f"| {category} | {err_msg} | " + f"{format_known_deviations[err_msg] if category == TestResult.KNOWN_DEVIATION else ''} | " + f"{count} |" + for (category, err_msg), count in format_results.most_common() + if category != TestResult.SUCCESS + ] + table_lines += ["", "
"] + sections.append("\n".join(table_lines)) + + format_stale_deviations = stale_deviations[format_name] + if format_stale_deviations: + stale_lines = [ + "#### Stale known deviations :x:", + "These known deviations did not occur. Remove them from the known deviations file.", + "", + "| Group | Issue |", + "|---|---|", + ] + stale_lines += [f"| {group} | {issue} |" for group, issue in format_stale_deviations.items()] + sections.append("\n".join(stale_lines)) output_file.write_text("\n\n".join(sections)) -def main(example_path_str: str, output_path_str: Optional[str], summary_path_str: Optional[str]) -> None: +def main( + example_path_str: str, + output_path_str: Optional[str], + summary_path_str: Optional[str], + known_deviations_path_str: Optional[str], +) -> None: example_path = Path(example_path_str) output_path = Path(output_path_str) if output_path_str else None summary_path = Path(summary_path_str) if summary_path_str else None + known_deviations_path = Path(known_deviations_path_str) if known_deviations_path_str else KNOWN_DEVIATIONS_FILE if not example_path.exists(): logger.error(f"Provided path for examples does not exist: {example_path}") sys.exit(1) + # A missing default file means that there are no known deviations, a missing file passed explicitly is an error + if known_deviations_path_str is None and not known_deviations_path.exists(): + known_deviations: dict[str, dict[str, str]] = {format_name: {} for format_name in FORMATS} + else: + try: + known_deviations = load_known_deviations(known_deviations_path) + except (OSError, tomllib.TOMLDecodeError, ValueError) as ex: + logger.error(f"Invalid known deviations file {known_deviations_path}: {ex}") + sys.exit(1) + if output_path: output_path.mkdir(parents=True, exist_ok=True) @@ -262,11 +364,18 @@ def main(example_path_str: str, output_path_str: Optional[str], summary_path_str ] test_results: dict[str, Counter[tuple[TestResult, str]]] = dict() + stale_deviations: dict[str, dict[str, str]] = dict() total_failed = 0 + run_failed = False for name, directory, glob_pattern, test_fn in test_configs: + test_files = sorted(directory.glob(f"**/{glob_pattern}")) + if not test_files: + logger.error(f"No {name} example files found in {directory}") + run_failed = True + format_results: Counter[tuple[TestResult, str]] = Counter() - for test_file in sorted(directory.glob(f"**/{glob_pattern}")): - test_result = test_fn(test_file, directory, output_path) + for test_file in test_files: + test_result = test_fn(test_file, directory, output_path, known_deviations[name]) format_results[test_result] += 1 test_results[name] = format_results @@ -281,9 +390,16 @@ def main(example_path_str: str, output_path_str: Optional[str], summary_path_str else: logger.error(f"Failed {failed:d}/{total:d} {name.lower()} tests") + occurred_deviations = {err_msg for result, err_msg in format_results if result == TestResult.KNOWN_DEVIATION} + stale_deviations[name] = {group: issue for group, issue in known_deviations[name].items() + if group not in occurred_deviations} + for group, issue in stale_deviations[name].items(): + logger.error(f"Stale known {name} deviation did not occur: {group} (see {issue})") + run_failed = True + if summary_path is not None: - summarize_output(test_results, example_path, summary_path) - sys.exit(1 if total_failed > 0 else 0) + summarize_output(test_results, known_deviations, stale_deviations, example_path, summary_path) + sys.exit(1 if total_failed > 0 or run_failed else 0) if __name__ == '__main__': @@ -292,6 +408,9 @@ def main(example_path_str: str, output_path_str: Optional[str], summary_path_str parser.add_argument("--examples", required=True, help="path to examples directory") parser.add_argument("--output", required=False, help="path to output directory") parser.add_argument("--summary", required=False, help="file to write summary to") + parser.add_argument("--known-deviations", required=False, + help=f"TOML file that lists the known deviations " + f"(default: {KNOWN_DEVIATIONS_FILE.name} next to this script)") args = parser.parse_args() - main(args.examples, args.output, args.summary) + main(args.examples, args.output, args.summary, args.known_deviations) diff --git a/etc/sdk-testdatagen/known_deviations.toml b/etc/sdk-testdatagen/known_deviations.toml new file mode 100644 index 000000000..54cd40a3a --- /dev/null +++ b/etc/sdk-testdatagen/known_deviations.toml @@ -0,0 +1,16 @@ +# Known deviations of the BaSyx Python SDK from the AAS specification, see `Readme.md` in this directory. +# +# Each entry marks one error group (as shown in the "Group" column of the summary) of a format as known deviation. +# Entries are listed separately per format in `[[JSON]]` and `[[XML]]` tables, which have exactly two keys: +# - `group`: the exact message of the error group +# - `issue`: a link to the issue that describes the deviation in detail +# Briefly state the reason for the deviation in a comment above each entry, for example: +# +# # Python's datetime does not support negative years +# [[XML]] +# group = "Negative dates are not supported: Python stdlib datetime requires year >= 1. Report at ..." +# issue = "https://github.com/eclipse-basyx/basyx-python-sdk/issues/" +# +# The test script fails if a listed known deviation does not occur (stale known deviation). + +# SDK only accepts an ExternalReference for EmbeddedDataSpecification/dataSpecification, examples use a ModelReference From 99d7cded7a2b8de4d17f9a061c8140373e02e1da Mon Sep 17 00:00:00 2001 From: hpoeche Date: Thu, 22 Oct 2026 17:56:25 +0200 Subject: [PATCH 14/14] Add known deviations for upstream dataSpecification error (#567) --- etc/sdk-testdatagen/known_deviations.toml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/etc/sdk-testdatagen/known_deviations.toml b/etc/sdk-testdatagen/known_deviations.toml index 54cd40a3a..583ec7b9b 100644 --- a/etc/sdk-testdatagen/known_deviations.toml +++ b/etc/sdk-testdatagen/known_deviations.toml @@ -13,4 +13,12 @@ # # The test script fails if a listed known deviation does not occur (stale known deviation). -# SDK only accepts an ExternalReference for EmbeddedDataSpecification/dataSpecification, examples use a ModelReference +# Upstream testdatagen error: examples use a ModelReference for EmbeddedDataSpecification/dataSpecification +[[JSON]] +group = "Expected a reference of type , got !" +issue = "https://github.com/eclipse-aascw/aas-core-meta/issues/394" + +# Upstream testdatagen error: examples use a ModelReference for EmbeddedDataSpecification/dataSpecification +[[XML]] +group = "{https://admin-shell.io/aas/3/1}dataSpecification is of type , expected !" +issue = "https://github.com/eclipse-aascw/aas-core-meta/issues/394"