diff --git a/.github/workflows/acceptance.yml b/.github/workflows/acceptance.yml new file mode 100644 index 00000000..f7648647 --- /dev/null +++ b/.github/workflows/acceptance.yml @@ -0,0 +1,70 @@ +name: Acceptance tests + +# Builds the in-repo dogfood plugins and runs the acceptance (golden-file) +# self-tests via CTest. This is independent of Tracktion's main private build +# pipeline; it exists so the `pluginval test` feature has a public, reproducible +# regression check across the three desktop platforms. + +on: + push: + branches: [ master, develop, v2 ] + pull_request: + workflow_dispatch: + +concurrency: + group: acceptance-${{ github.ref }} + cancel-in-progress: true + +jobs: + acceptance: + name: ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ ubuntu-latest, macos-latest, windows-latest ] + + env: + # Cache CPM's downloads (JUCE etc.) so reruns don't re-fetch them. + CPM_SOURCE_CACHE: ${{ github.workspace }}/.cpm-cache + + steps: + - uses: actions/checkout@v4 + + - name: Cache CPM sources + uses: actions/cache@v4 + with: + path: ${{ github.workspace }}/.cpm-cache + key: cpm-${{ matrix.os }}-${{ hashFiles('CMakeLists.txt', 'cmake/CPM.cmake') }} + restore-keys: cpm-${{ matrix.os }}- + + - name: Install Linux dependencies + if: runner.os == 'Linux' + run: | + sudo apt-get update + sudo apt-get install -y \ + libasound2-dev libx11-dev libxext-dev libxi-dev libxinerama-dev libxrandr-dev \ + libxcursor-dev libxcomposite-dev libfreetype6-dev libfontconfig1-dev \ + libgl1-mesa-dev libcurl4-openssl-dev ladspa-sdk ninja-build xvfb + + # The acceptance tests host the dogfood plugins via JUCE's own VST3 hosting, + # so the embedded VST3 validator (and its heavy SDK build) isn't needed here. + - name: Configure + run: > + cmake -B build + -DCMAKE_BUILD_TYPE=Release + -DPLUGINVAL_BUILD_TEST_PLUGINS=ON + -DPLUGINVAL_VST3_VALIDATOR=OFF + + - name: Build + run: cmake --build build --config Release + + - name: Run acceptance tests (Linux) + if: runner.os == 'Linux' + working-directory: build + run: xvfb-run --auto-servernum ctest -C Release --output-on-failure -R pluginval.acceptance + + - name: Run acceptance tests (macOS / Windows) + if: runner.os != 'Linux' + working-directory: build + run: ctest -C Release --output-on-failure -R pluginval.acceptance diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index fa5d16df..fbb9dcb0 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -67,7 +67,7 @@ jobs: run: | sudo add-apt-repository ppa:ubuntu-toolchain-r/test sudo apt-get update - sudo apt-get install -y freeglut3-dev g++ libasound2-dev libcurl4-openssl-dev libfreetype6-dev libjack-jackd2-dev libx11-dev libxcomposite-dev libxcursor-dev libxinerama-dev libxrandr-dev mesa-common-dev ladspa-sdk libwebkit2gtk-4.1-dev xvfb gdb + sudo apt-get install -y freeglut3-dev g++ libasound2-dev libcurl4-openssl-dev libfreetype6-dev libjack-jackd2-dev libx11-dev libxi-dev libxcomposite-dev libxcursor-dev libxinerama-dev libxrandr-dev mesa-common-dev ladspa-sdk libwebkit2gtk-4.1-dev xvfb gdb sudo /usr/bin/Xvfb $DISPLAY & - name: Make VST2 SDK available @@ -104,7 +104,7 @@ jobs: run: | pwd ls -ahl - ${{ matrix.test-binary }} --run-tests + ${{ matrix.test-binary }} run-tests - name: Cache JUCE example plugin binaries id: cache-plugins @@ -127,9 +127,9 @@ jobs: - name: Validate JUCE Plugin examples (VST3) shell: bash run: | - # Paths must be single quoted for bash not to escape the Windows backslash character \ used in absolute paths - ${{ env.APP_DIR }}/${{ matrix.test-binary }} --strictness-level 10 --validate '${{ env.PLUGIN_CACHE_PATH }}/DSPModulePluginDemo_artefacts/Release/VST3/DSPModulePluginDemo.vst3' - ${{ env.APP_DIR }}/${{ matrix.test-binary }} --strictness-level 10 --validate '${{ env.PLUGIN_CACHE_PATH }}/MultiOutSynthPlugin_artefacts/Release/VST3/MultiOutSynthPlugin.vst3' + # Paths must be single quoted for bash not to escape the Windows backslash character \ used in absolute paths + ${{ env.APP_DIR }}/${{ matrix.test-binary }} validate --strictness-level 10 '${{ env.PLUGIN_CACHE_PATH }}/DSPModulePluginDemo_artefacts/Release/VST3/DSPModulePluginDemo.vst3' + ${{ env.APP_DIR }}/${{ matrix.test-binary }} validate --strictness-level 10 '${{ env.PLUGIN_CACHE_PATH }}/MultiOutSynthPlugin_artefacts/Release/VST3/MultiOutSynthPlugin.vst3' - name: Validate JUCE Plugin examples (AU) shell: bash @@ -141,6 +141,8 @@ jobs: mkdir -p ~/Library/Audio/Plug-Ins/Components/ cp -R ${{ env.PLUGIN_CACHE_PATH }}/DSPModulePluginDemo_artefacts/Release/AU/DSPModulePluginDemo.component ~/Library/Audio/Plug-Ins/Components/ killall -9 AudioComponentRegistrar # kick the AU registrar + # Intentionally uses the deprecated "--validate" flat flag to keep the + # backwards-compatible alias covered (this step is continue-on-error). ${{ env.APP_DIR }}/${{ matrix.test-binary }} --strictness-level 10 --validate ~/Library/Audio/Plug-Ins/Components/DSPModulePluginDemo.component - name: Codesign (macOS) @@ -175,7 +177,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, macos-15, windows-latest] - juce: ["8.0.13"] + juce: ["8.0.13", "9.0.3"] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 @@ -184,7 +186,7 @@ jobs: if: runner.os == 'Linux' run: | sudo apt-get update - sudo apt-get install -y libasound2-dev libfreetype6-dev libx11-dev libxcomposite-dev libxcursor-dev libxinerama-dev libxrandr-dev mesa-common-dev libwebkit2gtk-4.1-dev ladspa-sdk + sudo apt-get install -y libasound2-dev libfreetype6-dev libx11-dev libxi-dev libxcomposite-dev libxcursor-dev libxinerama-dev libxrandr-dev mesa-common-dev libwebkit2gtk-4.1-dev ladspa-sdk - name: Configure run: cmake -B build -S tests/pluginval_as_dependency -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE}} -DJUCE_VERSION="${{ matrix.juce }}" diff --git a/.gitignore b/.gitignore index e2c58172..355bd37e 100644 --- a/.gitignore +++ b/.gitignore @@ -79,4 +79,6 @@ pluginval.build/ pluginval.xcodeproj/ pluginval_artefacts/ -.claude/*local* \ No newline at end of file +.claude/*local* +# Acceptance-test failure diffs (written next to the reference) +*-diff.wav diff --git a/CHANGELIST.md b/CHANGELIST.md index fb4e7fd6..b53db341 100644 --- a/CHANGELIST.md +++ b/CHANGELIST.md @@ -1,5 +1,21 @@ # pluginval Change List +### 2.0.0 +- Updated JUCE to 9.0.3 +- Fixed the crash handler so a backtrace is shown again when a plugin crashes, and made it async-signal-safe [#144] +- Added acceptance testing: `pluginval test ` renders a plugin with a known state, input (audio/MIDI) and optional playhead, then compares the output against a checked-in reference WAV, exiting `0` on a match and `1` on a mismatch (with a diff WAV). Use `--record-missing` to record new references. See `docs/Acceptance testing.md` +- Restructured the command line into subcommands: `pluginval validate [options] ` (the default), `pluginval run-tests` and `pluginval strictness-help [level]`. The plugin path is now a positional argument of `validate` +- **Deprecated:** the flat flags `--validate `, `--run-tests` and `--strictness-help [level]` still work as aliases (with a one-line notice) but will be removed in a future version. `pluginval ` remains a silent shorthand for `validate` +- Replaced the hand-rolled command-line parser with CLI11 and a single JSON-based settings pipeline +- Added `--config ` to load settings from JSON (repeatable; later files win per key) +- Settings precedence is now (lowest to highest): defaults, environment variables, `--config`, command-line options +- Environment variable support is now automatic for every option (including `DISABLED_TESTS`) +- **Breaking:** unrecognised options now produce an error instead of being ignored +- **Breaking:** invalid option values now error instead of silently falling back (e.g. `--rtcheck banana`, `--strictness-level abc`) +- **Breaking:** falsey flag environment variables now mean off (e.g. `SKIP_GUI_TESTS=0` no longer enables skipping) +- `--help` output is now generated by CLI11 +- Enabled the editor stress and extreme (real-time allocation / oversized block) tests that were present in the source tree but had not been compiled into the build + ### 1.0.5 - Added drag-and-drop of plug-in files onto the main window, with two drop zones to either validate the plug-in or add it to the plugin list [#170] - Added static linking to the Windows runtime so it should run on more Windows systems (particularly non-dev machines) diff --git a/CLAUDE.md b/CLAUDE.md index fcec6889..ff21a61e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -4,9 +4,9 @@ **pluginval** is a cross-platform audio plugin validator and tester application developed by Tracktion Corporation. It tests VST, VST3, AU (Audio Unit), LV2, and LADSPA plugins for compatibility and stability with host applications. -- **Version**: 1.0.4 (see `VERSION` file) +- **Version**: 2.0.0 (see `VERSION` file) - **License**: GPLv3 -- **Framework**: Built on JUCE (v8.0.x) +- **Framework**: Built on JUCE (v9.0.x) - **Language**: C++20 ### Key Features @@ -75,21 +75,30 @@ Replace `` with the ID from step 2. ``` pluginval/ -├── Source/ # Main application source code +├── source/ # Main application source code │ ├── Main.cpp # Application entry point │ ├── MainComponent.cpp/h # GUI main window component │ ├── Validator.cpp/h # Core validation orchestration │ ├── PluginTests.cpp/h # Test framework and base classes -│ ├── CommandLine.cpp/h # CLI argument parsing +│ ├── CommandLine.cpp/h # Thin CLI adapter (delegates to SettingsParser) +│ ├── PluginvalSettings.h # Unified settings struct + JSON mapping + toPluginTestOptions() +│ ├── SettingsParser.cpp/h # CLI/env/config -> merged JSON -> settings; child-process handoff +│ ├── SettingsSerializer.cpp/h # JSON load/save + value coercions (comma lists, hex seed) │ ├── CrashHandler.cpp/h # Crash reporting utilities │ ├── TestUtilities.cpp/h # Helper functions for tests │ ├── RTCheck.h # Real-time safety checking macros │ ├── PluginvalLookAndFeel.h # Custom UI styling │ ├── StrictnessInfoPopup.h # Strictness level info UI │ ├── binarydata/ # Binary resources (icons) +│ ├── native/ # Platform helpers (pluginval_native_posix.h: async-signal-safe stack trace writing) │ ├── vst3validator/ # Embedded VST3 validator integration │ │ ├── VST3ValidatorRunner.h │ │ └── VST3ValidatorRunner.cpp +│ ├── acceptance/ # `pluginval test` golden-file subsystem (parallel to validate) +│ │ ├── TestConfig.cpp/h # snake_case JSON config struct (independent of PluginvalSettings) +│ │ ├── AcceptanceTest.cpp/h # plugin load + state + input + render; record-or-compare orchestrator +│ │ ├── ReferenceComparator.cpp/h # Comparator interface + registry + SampleComparator +│ │ └── TestReporter.cpp/h # text + JSON result, exit code │ └── tests/ # Individual test implementations │ ├── BasicTests.cpp # Core plugin tests (info, state, audio) │ ├── BusTests.cpp # Audio bus configuration tests @@ -105,6 +114,10 @@ pluginval/ ├── tests/ │ ├── AddPluginvalTests.cmake # CMake module for CTest integration │ ├── test_plugins/ # Test plugin files +│ │ ├── tone_generator/ # Deterministic dogfood generator (PLUGINVAL_BUILD_TEST_PLUGINS) +│ │ ├── gain/ # Deterministic dogfood gain effect (for the input.audio path) +│ │ └── playhead_probe/ # Writes the host transport to output (for the playhead path) +│ ├── acceptance/ # Acceptance self-tests: configs (*.json.in), inputs/, checked-in refs/ WAVs │ ├── mac_tests/ # macOS-specific tests │ └── windows_tests.bat # Windows test scripts ├── docs/ # Documentation @@ -147,6 +160,7 @@ cmake --build Builds/Debug --config Debug | `WITH_ADDRESS_SANITIZER` | Enable AddressSanitizer | OFF | | `WITH_THREAD_SANITIZER` | Enable ThreadSanitizer | OFF | | `VST2_SDK_DIR` | Path to VST2 SDK (env var) | - | +| `PLUGINVAL_BUILD_TEST_PLUGINS` | Build the in-repo dogfood plugins + acceptance CTest self-tests | OFF | ### Enabling VST2 Support @@ -188,6 +202,120 @@ VST2_SDK_DIR=/path/to/vst2sdk cmake -B Builds/Debug . - Auto-registers via static instance pattern - Defines requirements (thread, GUI needs) +### CLI Settings Pipeline + +#### Subcommand dispatch layer + +The command line is structured into subcommands, peeled off by a thin verb +dispatcher in front of the settings pipeline (it does **not** use CLI11-native +subcommands, so the env/config/CLI layering below is untouched): + +- `pluginval validate [options] ` — the default; `` is a + positional argument. `pluginval ` and `pluginval [options] ` + (no verb) also resolve to validate. +- `pluginval run-tests` — runs the internal unit tests. +- `pluginval strictness-help [level]` — lists tests at a strictness level. + +`settings_parser::dispatch()` (in `SettingsParser.cpp`) takes the `tokenise()`d +command line and returns a `DispatchResult { command, validateTokens, +deprecatedAlias, strictnessLevel }`. For `validate` it strips the verb and hands +`validateTokens` to `parseTokens` unchanged. `CommandLine.cpp`'s +`performCommandLine()` switches on `command` and emits a one-line stderr notice +when `deprecatedAlias` is set. + +The old flat flags (`--validate `, `--run-tests`, `--strictness-help`) +are kept as **deprecated aliases** that route to the same commands with +`deprecatedAlias = true` (the bare-path shorthand and the internal child handoff +stay silent). `preprocess()` is now `tokenise()` + `insertImplicitValidate()`. +The child process is launched with the explicit verb: +`validate --config-base64 `. + +Note: `--config` is parsed manually (it is stripped from the tokens fed to the +CLI11 pass) so its greedy CLI11 vector parsing can't swallow the positional +plugin path; it stays registered only so it appears in `--help`. + +#### Settings layering + +Command-line parsing centres on one plain settings struct (`PluginvalSettings`) +that CLI11 binds to directly. A single instance is filled by successive layers, +**lowest to highest precedence: defaults → environment → `--config` → CLI** +(in `SettingsParser::parseTokens`): + +1. **preprocess** the raw command line — rewrite the deprecated `strictnessLevel`, + strip the macOS `-NSDocumentRevisionsDebugMode YES` flag, and insert an + implicit `--validate` when the last argument is a bare plugin path. +2. **Environment layer.** Env-var names are *derived* from the registered + options (`--strictness-level` → `STRICTNESS_LEVEL`), so there is no separate + env table. A synthetic `--name=value` argv is built from the environment and + parsed by CLI11, reusing all its coercion. +3. **`--config` layer.** Repeatable; each JSON file is `merge_patch`-ed in + command-line order (later files win per key). Beats the environment. +4. **CLI layer.** The real arguments are parsed last and beat everything; + CLI11 only overwrites a member when its option was actually provided. + +`configureApp()` registers every option (bound to the struct) and is used for +both the env pass and the CLI pass. Comma lists use `->delimiter(',')`, the enum +uses a `CheckedTransformer`, and the hex/int seed is a small callback. +`PluginvalSettings::toPluginTestOptions()` converts to the JUCE-flavoured +`PluginTests::Options` at the boundary. + +Adding a new option is three edits: a struct member, an entry in the nlohmann +macro list, and one `add_option(...)` line — its environment variable then works +automatically. `SettingsSerializer` handles JSON load/save plus the two +remaining conversions (hex seed, disabled-tests file). + +The child validation process receives a fully-resolved, **authoritative** +settings set via a base64-encoded JSON argument (`--config-base64`), avoiding +per-flag re-serialisation and command-line quoting hazards. `--help`/`--version` +are handled by CLI11 (auto usage + a footer with the env-var/commands notes). + +### Acceptance Testing (`pluginval test`) + +A **parallel subsystem** to validate, in `source/acceptance/`. It answers "does +this plugin produce the expected output for a known input + state?" — a +deterministic *render + golden-file comparison*, not a unit-test pass/fail. Full +spec: `tests/acceptance/Acceptance testing design.md`; end-user guide: +`docs/Acceptance testing.md`. + +- **CLI**: `pluginval test `. A new `Command::test` is recognised + by `settings_parser::dispatch()` (captures the positional config path into + `DispatchResult::testConfigPath`, plus `recordMissing` / `timeoutMs` from + `--record-missing` / `--timeout-ms` or `RECORD_MISSING` / `TIMEOUT_MS`), + `isCommandLine()` and `getFooterText()`; `CommandLine.cpp`'s + `performCommandLine()` has a `Command::test` branch that starts an + `acceptance::TestRunner`, which renders on a **background thread** (message + thread left free; plugin deleted on it), with a watchdog for the timeout, and + quits via `callAsync` when done. +- **Config**: `acceptance::TestConfig` (`TestConfig.cpp/h`) — std-typed struct, + **snake_case JSON keys** mapped via explicit `to_json`/`from_json` (members + stay camelCase). It is **independent** of `PluginvalSettings` / the `--config` + layering: the test config is a positional argument loaded standalone. +- **Flow** (`AcceptanceTest.cpp`): load plugin → apply `state.file` then + `state.parameters` (normalised, applied in file order via `ordered_json`, + matched by index / case-insensitive `HostedParameter` ID or name) → feed `input.audio`/`input.midi` or silence → if a `playhead` is + configured, point a fixed-tempo transport (`FixedPlayHead`, position advances + per block) at the plugin → render a fixed duration block-by-block (reusing the + `AudioProcessingTest` shape + the VST3-safe helpers in `TestUtilities.h`). If + no reference exists it fails, or with `--record-missing` **records** one + (32-bit float WAV + `.wav.json` sidecar manifest, whose `config_hash` is + XXH3 of the config minus paths, with input/state files hashed by content); + otherwise it **compares** and writes a diff WAV on failure. Exit `0`/`1`. +- **Comparators** (`ReferenceComparator.cpp/h`): pluggable `Comparator` + + `createComparator(name)` registry. v1 ships only `sample` (per-sample abs-diff + tolerance, default one 16-bit LSB = `1/32768`; `0` = bit-exact). Adding + `spectrum`/`crosscorr`/etc. is one registry entry, no config/runner changes. +- **Dogfood + self-tests**: three minimal deterministic `juce_add_plugin` targets + behind `PLUGINVAL_BUILD_TEST_PLUGINS` — `tests/test_plugins/tone_generator/` + (closed-form sine/square generator, phase resets on `prepareToPlay`), + `tests/test_plugins/gain/` (a gain effect, dogfoods the `input.audio` path) and + `tests/test_plugins/playhead_probe/` (writes the host transport to its output, + dogfoods the `playhead` path). `tests/acceptance/` holds checked-in configs + (`*.json.in`, the plugin paths + input dir substituted at configure time), + `inputs/` and reference WAVs, run via CTest (`pluginval.acceptance.*`: sine-440, + square-220, square-state, gain-half, playhead-120). Phase 2 items (config-array + multiplexing, automation, extra comparators, child-process isolation) are notes + only. + ### Test Framework Tests are self-registering. To find all tests, look for static instances: @@ -228,12 +356,12 @@ The VST3 validator (Steinberg's vstvalidator) is embedded into pluginval when bu 1. The VST3 SDK is fetched via CPM during CMake configure 2. The SDK's own `validator` target is built as a separate executable 3. A CMake script (`cmake/GenerateBinaryHeader.cmake`) converts the compiled binary into a C byte array header -4. `VST3ValidatorRunner` (`Source/vst3validator/`) extracts the embedded binary to a temp file on first use +4. `VST3ValidatorRunner` (`source/vst3validator/`) extracts the embedded binary to a temp file on first use 5. When the `VST3validator` test runs, it spawns the extracted validator as a subprocess **Key files:** - `cmake/GenerateBinaryHeader.cmake` — binary-to-C-header conversion script -- `Source/vst3validator/VST3ValidatorRunner.h/cpp` — extracts embedded binary, returns `juce::File` +- `source/vst3validator/VST3ValidatorRunner.h/cpp` — extracts embedded binary, returns `juce::File` **Disabling embedded validator:** ```bash @@ -322,14 +450,20 @@ ut.logVerboseMessage("Detail message"); // Only with --verbose flag Basic usage: ```bash -./pluginval --strictness-level 5 /path/to/plugin.vst3 +./pluginval validate --strictness-level 5 /path/to/plugin.vst3 ``` -Key options: -- `--validate [path]` - Validate plugin at path +Commands: +- `validate [options] ` - Validate the plugin at the given path/AU id (the default; `./pluginval ` also works) +- `run-tests` - Run the internal unit tests +- `strictness-help [level]` - List the tests that run at a strictness level + +The flat flags `--validate `, `--run-tests` and `--strictness-help [level]` are deprecated aliases. + +Key options (for `validate`): +- `--config [file.json]` - Load a full settings set from JSON (overridden by env vars and CLI options) - `--strictness-level [1-10]` - Test thoroughness (default: 5) - `--skip-gui-tests` - Skip GUI tests (for headless CI) -- `--validate-in-process` - Don't use child process (for debugging) - `--timeout-ms [ms]` - Test timeout (default: 30000, -1 for none) - `--verbose` - Enable verbose logging - `--output-dir [dir]` - Directory for log files @@ -374,6 +508,8 @@ add_pluginval_tests(MyPluginTarget ### External - **JUCE** (v8.0.x) - Audio application framework (git submodule) - **magic_enum** (v0.9.7) - Enum reflection (fetched via CPM) +- **CLI11** (v2.6.2) - CLI argument parsing, header-only (fetched via CPM) +- **nlohmann/json** (3.12.0) - JSON settings layering/serialisation (fetched via CPM) - **rtcheck** (optional, macOS) - Real-time safety checking (fetched via CPM) - **VST3 SDK** (v3.7.x) - Steinberg VST3 SDK for embedded validator (fetched via CPM, optional) @@ -394,7 +530,7 @@ Debug unit tests run automatically in debug builds: Run internal tests via CLI: ```bash -./pluginval --run-tests +./pluginval run-tests ``` ## Release Process @@ -408,7 +544,7 @@ Run internal tests via CLI: ## Common Tasks for AI Assistants ### Finding Where Tests Are Defined -- All test classes are in `Source/tests/*.cpp` +- All test classes are in `source/tests/*.cpp` - Search for `static.*Test.*Test;` to find registrations - Each test subclasses `PluginTest` @@ -439,6 +575,6 @@ Run internal tests via CLI: - Always test changes on multiple platforms when possible - VST3 plugins have specific threading requirements - use the `*OnMessageThreadIfVST3` helpers -- Child process validation is the default and recommended for production use -- In-process validation (`--validate-in-process`) is useful for debugging but a crashing plugin will crash pluginval +- The GUI runs each validation in a separate child process for crash isolation (the default) +- The CLI `--validate` path runs in-process; a crashing plugin will terminate pluginval, and the signal handler reports it as a failure rather than a pass - Real-time safety checking is only available on macOS currently (uses rtcheck library) diff --git a/CMakeLists.txt b/CMakeLists.txt index f2ac5bb7..e682e9c6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -28,6 +28,10 @@ endif() option(WITH_ADDRESS_SANITIZER "Enable Address Sanitizer" OFF) option(WITH_THREAD_SANITIZER "Enable Thread Sanitizer" OFF) +# Builds the in-repo dogfood plugins (e.g. the tone generator) used by the +# acceptance self-tests. Off for normal release builds. +option(PLUGINVAL_BUILD_TEST_PLUGINS "Build the in-repo test plugins used by the acceptance self-tests" OFF) + message(STATUS "Sanitizers: ASan=${WITH_ADDRESS_SANITIZER} TSan=${WITH_THREAD_SANITIZER}") if (WITH_ADDRESS_SANITIZER) if (MSVC) @@ -61,6 +65,15 @@ option(JUCE_ENABLE_MODULE_SOURCE_GROUPS "Enable Module Source Groups" ON) include(cmake/CPM.cmake) CPMAddPackage("gh:Neargye/magic_enum#v0.9.7") +CPMAddPackage("gh:CLIUtils/CLI11@2.6.2") +CPMAddPackage("gh:nlohmann/json@3.12.0") + +# xxHash is used header-only (XXH_INLINE_ALL) for the acceptance-test config hash +CPMAddPackage( + NAME xxHash + GITHUB_REPOSITORY Cyan4973/xxHash + VERSION 0.8.3 + DOWNLOAD_ONLY YES) if(PLUGINVAL_ENABLE_RTCHECK) CPMAddPackage("gh:Tracktion/rtcheck#main") @@ -68,7 +81,7 @@ endif() # Only fetch JUCE when top-level (when used as dependency, JUCE is already available) if (pluginval_IS_TOP_LEVEL) - CPMAddPackage("gh:juce-framework/juce#8.0.13") + CPMAddPackage("gh:juce-framework/juce#9.0.3") endif() @@ -103,7 +116,7 @@ endif() juce_add_gui_app(pluginval BUNDLE_ID com.Tracktion.pluginval COMPANY_NAME Tracktion - ICON_BIG "${CMAKE_CURRENT_SOURCE_DIR}/Source/binarydata/icon.png" + ICON_BIG "${CMAKE_CURRENT_SOURCE_DIR}/source/binarydata/icon.png" HARDENED_RUNTIME_ENABLED TRUE HARDENED_RUNTIME_OPTIONS com.apple.security.cs.allow-unsigned-executable-memory com.apple.security.cs.disable-library-validation com.apple.security.get-task-allow) @@ -116,35 +129,50 @@ set_target_properties(pluginval PROPERTIES CXX_VISIBILITY_PRESET hidden) set(SourceFiles - Source/CommandLine.h - Source/CrashHandler.h - Source/MainComponent.h - Source/PluginTests.h - Source/TestUtilities.h - Source/Validator.h - Source/CommandLine.cpp - Source/CrashHandler.cpp - Source/Main.cpp - Source/MainComponent.cpp - Source/PluginTests.cpp - Source/tests/BasicTests.cpp - Source/tests/LocaleTest.cpp - Source/tests/BusTests.cpp - Source/tests/ParameterFuzzTests.cpp - Source/TestUtilities.cpp - Source/Validator.cpp) + source/CommandLine.h + source/CrashHandler.h + source/MainComponent.h + source/PluginTests.h + source/PluginvalSettings.h + source/SettingsParser.h + source/SettingsSerializer.h + source/TestUtilities.h + source/Validator.h + source/acceptance/TestConfig.h + source/acceptance/TestConfig.cpp + source/acceptance/AcceptanceTest.h + source/acceptance/AcceptanceTest.cpp + source/acceptance/ReferenceComparator.h + source/acceptance/ReferenceComparator.cpp + source/acceptance/TestReporter.h + source/acceptance/TestReporter.cpp + source/CommandLine.cpp + source/CrashHandler.cpp + source/Main.cpp + source/MainComponent.cpp + source/PluginTests.cpp + source/SettingsParser.cpp + source/SettingsSerializer.cpp + source/tests/BasicTests.cpp + source/tests/LocaleTest.cpp + source/tests/BusTests.cpp + source/tests/EditorTests.cpp + source/tests/ExtremeTests.cpp + source/tests/ParameterFuzzTests.cpp + source/TestUtilities.cpp + source/Validator.cpp) target_sources(pluginval PRIVATE ${SourceFiles}) -source_group(TREE ${CMAKE_CURRENT_SOURCE_DIR}/Source PREFIX Source FILES ${SourceFiles}) +source_group(TREE ${CMAKE_CURRENT_SOURCE_DIR}/source PREFIX Source FILES ${SourceFiles}) # Add VST3 validator runner sources to pluginval (extracts embedded binary at runtime) if(PLUGINVAL_VST3_VALIDATOR) set(VST3ValidatorFiles - Source/vst3validator/VST3ValidatorRunner.h - Source/vst3validator/VST3ValidatorRunner.cpp + source/vst3validator/VST3ValidatorRunner.h + source/vst3validator/VST3ValidatorRunner.cpp ) target_sources(pluginval PRIVATE ${VST3ValidatorFiles}) - source_group("Source/vst3validator" FILES ${VST3ValidatorFiles}) + source_group("source/vst3validator" FILES ${VST3ValidatorFiles}) endif() if (DEFINED ENV{VST2_SDK_DIR}) @@ -176,7 +204,11 @@ target_link_libraries(pluginval PRIVATE juce::juce_audio_processors juce::juce_audio_utils juce::juce_recommended_warning_flags - magic_enum) + magic_enum + CLI11::CLI11 + nlohmann_json::nlohmann_json) + +target_include_directories(pluginval PRIVATE "${xxHash_SOURCE_DIR}") if (${CMAKE_SYSTEM_NAME} MATCHES "Linux") target_link_libraries(pluginval PRIVATE @@ -234,13 +266,26 @@ else() set(PLUGINVAL_TARGET "${CMAKE_PROJECT_NAME}_VST3") endif() - get_target_property(artefact ${PLUGINVAL_TARGET} JUCE_PLUGIN_ARTEFACT_FILE) + # An umbrella/multi-product project has no ${CMAKE_PROJECT_NAME}_AU target, so skip instead of hard-erroring. + if (TARGET ${PLUGINVAL_TARGET}) + get_target_property(artefact ${PLUGINVAL_TARGET} JUCE_PLUGIN_ARTEFACT_FILE) + + # TODO: This doesn't populate the executable in clion + add_custom_target(${CMAKE_PROJECT_NAME}_pluginval_cli + COMMAND $ + validate + --strictness-level 10 + ${artefact} + DEPENDS pluginval ${PLUGINVAL_TARGET} + COMMENT "Run pluginval CLI with strict validation") + endif() +endif() - # TODO: This doesn't populate the executable in clion - add_custom_target(${CMAKE_PROJECT_NAME}_pluginval_cli - COMMAND $ - --validate ${artefact} - --strictness-level 10 - DEPENDS pluginval ${PLUGINVAL_TARGET} - COMMENT "Run pluginval CLI with strict validation") +# In-repo dogfood plugins + acceptance self-tests. +if (PLUGINVAL_BUILD_TEST_PLUGINS) + enable_testing() + add_subdirectory(tests/test_plugins/tone_generator) + add_subdirectory(tests/test_plugins/gain) + add_subdirectory(tests/test_plugins/playhead_probe) + add_subdirectory(tests/acceptance) endif() diff --git a/README.md b/README.md index 4ad3a92a..4daf1471 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ CPMAddPackage("gh:tracktion/pluginval#develop") Then all you need to do is call `add_subdirectory ("modules/pluginval")` in your `CMakeLists.txt`. This should be done **after** your call to `juce_add_plugin`. -Note that only JUCE 8 is currently supported/tested for this method. +Note that only JUCE 8 and 9 are currently supported/tested for this method. ### Third-party Installation ###### _Chocolatey (Windows):_ @@ -102,6 +102,7 @@ This means you can check the exit code on your various CI and mark builds a fail - [Testing plugins with pluginval]() - [Debugging a failed validation]() - [Adding pluginval to CI]() + - [Acceptance testing]() ### Contributing If you would like to contribute to the project please do! It's very simple to add tests, simply: diff --git a/ROADMAP.md b/ROADMAP.md index e7d181b5..5a63ca77 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -5,7 +5,7 @@ There are lots of ways pluginval can be improved, some of these are listed below You can help get there by issuing PRs or [funding](FUNDING.md) development. - [x] Integration of real-time safety checking (via rtcheck) -- [ ] Improved command-line handling and json config file support +- [x] Improved command-line handling and json config file support - [ ] Acceptance testing using input files and reference output files - [ ] Improved stack trace/crash reporting - [ ] Automatic integration of Asan/Tsan on platforms that allow it diff --git a/Source/CommandLine.cpp b/Source/CommandLine.cpp deleted file mode 100644 index 57548efc..00000000 --- a/Source/CommandLine.cpp +++ /dev/null @@ -1,653 +0,0 @@ -/*============================================================================== - - Copyright 2018 by Tracktion Corporation. - For more information visit www.tracktion.com - - You may also use this code under the terms of the GPL v3 (see - www.gnu.org/licenses). - - pluginval IS PROVIDED "AS IS" WITHOUT ANY WARRANTY, AND ALL WARRANTIES, WHETHER - EXPRESSED OR IMPLIED, INCLUDING MERCHANTABILITY AND FITNESS FOR PURPOSE, ARE - DISCLAIMED. - - ==============================================================================*/ - -#include "CommandLine.h" -#include "Validator.h" -#include "CrashHandler.h" -#include "PluginTests.h" - -#if JUCE_MAC - #include - #include -#endif - -#include - -//============================================================================== -static void exitWithError (const juce::String& error) -{ - std::cout << error << std::endl << std::endl; - juce::JUCEApplication::getInstance()->setApplicationReturnValue (1); - juce::JUCEApplication::getInstance()->quit(); -} - -static void hideDockIcon() -{ - #if JUCE_MAC - juce::Process::setDockIconVisible (false); - #endif -} - -inline std::mutex& getCoutMutex() -{ - static std::mutex m; - return m; -} - -inline void logLine (const juce::String& m) -{ - const std::scoped_lock sl (getCoutMutex()); - std::cout << m << std::endl; -} - -inline void logAndFlush (const juce::String& m) -{ - const std::scoped_lock sl (getCoutMutex()); - std::cout << m << std::flush; -} - -//============================================================================== -//============================================================================== -CommandLineValidator::CommandLineValidator() -{ -} - -CommandLineValidator::~CommandLineValidator() -{ -} - -void CommandLineValidator::validate (const juce::String& fileOrID, PluginTests::Options options) -{ - validator = std::make_unique (fileOrID, options, ValidationType::inProcess, - [] (auto id) - { - logLine ("Started validating: " + id); - }, - [] (auto, uint32_t exitCode) - { - if (exitCode > 0) - exitWithError ("*** FAILED"); - else - juce::JUCEApplication::getInstance()->quit(); - }, - [] (auto m) - { - logAndFlush (m); - }); -} - - -//============================================================================== -//============================================================================== -namespace -{ - juce::ArgumentList::Argument getArgumentAfterOption (const juce::ArgumentList& args, juce::StringRef option) - { - for (int i = 0; i < args.size() - 1; ++i) - if (args[i] == option) - return args[i + 1]; - - return {}; - } - - juce::var getOptionValue (const juce::ArgumentList& args, juce::StringRef option, juce::var defaultValue, juce::StringRef errorMessage) - { - if (args.containsOption (option)) - { - const auto nextArg = getArgumentAfterOption (args, option); - - if (nextArg.isShortOption() || nextArg.isLongOption()) - juce::ConsoleApplication::fail (errorMessage, -1); - - return nextArg.text; - } - - return defaultValue; - } - - int getStrictnessLevel (const juce::ArgumentList& args) - { - return juce::jlimit (1, 10, (int) getOptionValue (args, "--strictness-level", 5, "Missing strictness level argument! (Must be between 1 - 10)")); - } - - juce::int64 getRandomSeed (const juce::ArgumentList& args) - { - const juce::String seedString = getOptionValue (args, "--random-seed", "0", "Missing random seed argument!").toString(); - - if (! seedString.containsOnly ("x-0123456789acbdef")) - juce::ConsoleApplication::fail ("Invalid random seed argument!", -1); - - if (seedString.startsWith ("0x")) - return seedString.getHexValue64(); - - return seedString.getLargeIntValue(); - } - - juce::int64 getTimeout (const juce::ArgumentList& args) - { - return getOptionValue (args, "--timeout-ms", 30000, "Missing timeout-ms level argument!"); - } - - int getNumRepeats (const juce::ArgumentList& args) - { - return juce::jmax (1, (int) getOptionValue (args, "--repeat", 1, "Missing repeat argument! (Must be greater than 0)")); - } - - juce::File getDataFile (const juce::ArgumentList& args) - { - return getOptionValue (args, "--data-file", {}, "Missing data-file path argument!").toString(); - } - - juce::File getOutputDir (const juce::ArgumentList& args) - { - return getOptionValue (args, "--output-dir", {}, "Missing output-dir path argument!").toString(); - } - - juce::String getOutputFilename (const juce::ArgumentList& args) - { - return getOptionValue (args, "--output-filename", {}, "Missing output-filename path argument!").toString(); - } - - std::vector getSampleRates (const juce::ArgumentList& args) - { - juce::StringArray input = juce::StringArray::fromTokens (getOptionValue (args, - "--sample-rates", - juce::String ("44100,48000,96000"), - "Missing sample rate list argument!") - .toString(), - ",", - "\""); - std::vector output; - - for (juce::String sr : input) - output.push_back (sr.getDoubleValue()); - - return output; - } - - std::vector getBlockSizes (const juce::ArgumentList& args) - { - juce::StringArray input = juce::StringArray::fromTokens (getOptionValue (args, - "--block-sizes", - juce::String ("64,128,256,512,1024"), - "Missing block size list argument!") - .toString(), - ",", - "\""); - std::vector output; - - for (juce::String sr : input) - output.push_back (sr.getIntValue()); - - return output; - } - - juce::StringArray getDisabledTests (const juce::ArgumentList& args) - { - const auto value = getOptionValue (args, "--disabled-tests", {}, "Missing disabled-tests path argument!").toString(); - - if (juce::File::isAbsolutePath (value)) - { - const juce::File disabledTestsFile (value); - - juce::StringArray disabledTests; - disabledTestsFile.readLines (disabledTests); - - return disabledTests; - } - - return juce::StringArray::fromTokens (value, ",", ""); - } - - bool isPluginArgument (juce::String arg) - { - juce::AudioPluginFormatManager formatManager; - #if JUCE_VERSION >= 0x08000B - juce::addDefaultFormatsToManager (formatManager); - #else - formatManager.addDefaultFormats(); - #endif - - for (auto format : formatManager.getFormats()) - if (format->fileMightContainThisPluginType (arg)) - return true; - - // The above will check if the file actually exists which isn't really what we want for CLI parsing - if (auto f = juce::File::createFileWithoutCheckingPath (arg); - f.hasFileExtension (".vst3") - #if JUCE_PLUGINHOST_VST - || f.hasFileExtension (".dll") - #endif - ) - return true; - - return false; - } -} - -//============================================================================== -struct Option -{ - const char* name; - bool requiresValue; -}; - -static juce::String getEnvironmentVariableName (Option opt) -{ - return juce::String (opt.name).trimCharactersAtStart ("-").replace ("-", "_").toUpperCase(); -} - -static Option possibleOptions[] = -{ - { "--strictness-level", true }, - { "--random-seed", true }, - { "--timeout-ms", true }, - { "--verbose", true }, - { "--skip-gui-tests", false }, - { "--data-file", true }, - { "--output-dir", true }, - { "--output-filename", true }, - { "--repeat", true }, - { "--randomise", false }, - { "--sample-rates", true }, - { "--block-sizes", true }, - { "--rtcheck", false }, -}; - -static juce::StringArray mergeEnvironmentVariables (juce::StringArray args, std::function environmentVariableProvider = [] (const juce::String& name, const juce::String& defaultValue) { return juce::SystemStats::getEnvironmentVariable (name, defaultValue); }) -{ - for (auto arg : possibleOptions) - { - auto envVarName = getEnvironmentVariableName (arg); - auto envVarValue = environmentVariableProvider (envVarName, {}); - - if (envVarValue.isNotEmpty()) - { - const int index = args.indexOf (arg.name); - - if (index != -1) - { - std::cout << "Skipping environment variable " << envVarName << " due to " << arg.name << " set" << std::endl; - continue; - } - - if (arg.requiresValue) - args.insert (0, envVarValue); - - args.insert (0, arg.name); - } - } - - return args; -} - - -//============================================================================== -//============================================================================== -static juce::String getHelpMessage() -{ - const juce::String appName (juce::JUCEApplication::getInstance()->getApplicationName()); - const juce::String juceVersion (juce::SystemStats::getJUCEVersion()); - - return juce::String (R"(//============================================================================== -)" + appName + R"( -)" + juceVersion + R"( - -Description: - Validate plugins to test compatibility with hosts and verify plugin API conformance - -Usage: - --version - Print pluginval version. - --validate [pathToPlugin] - Validates the plugin at the given path. - N.B. the "--validate" flag is optional if the path is the last argument. - This enables you to validate a plugin with simply "pluginval path_to_plugin". - - --sample-rates [list of comma separated sample rates] - If specified, sets the list of sample rates at which tests will be executed - (default=44100,48000,96000) - --block-sizes [list of comma separated block sizes] - If specified, sets the list of block sizes at which tests will be executed - (default=64,128,256,512,1024) - --random-seed [hex or int] - Sets the random seed to use for the tests. Useful for replicating test - environments. - --data-file [pathToFile] - If specified, sets a path to a data file which can be used by tests to - configure themselves. This can be useful for things like known audio output. - - --strictness-level [1-10] - Sets the strictness level to use. A minimum level of 5 (also the default) - is recommended for compatibility. - Higher levels include longer, more thorough tests such as fuzzing. - --strictness-help [level] - Lists all tests that run at the given strictness level (default: 5). - --timeout-ms [numMilliseconds] - Sets a timout which will stop validation with an error if no output from any - test has happened for this number of ms. - By default this is 30s but can be set to "-1" (must be quoted) to never timeout. - --rtcheck [empty, disabled, enabled or relaxed] - Turns on real-time safety checks using rtcheck (macOS and Linux only). - relaxed mode doesn't run the checks for the first processing block as a lot of plugins - use this to allocate or initialise thread-locals (which can allocate) - - --repeat [num repeats] - If specified repeats the tests a given number of times. Note that this does - not delete and re-instantiate the plugin for each repeat. - --randomise - If specified, the tests are run in a random order per repeat. - - --skip-gui-tests - If specified, avoids tests that create GUI windows, which can cause problems - on headless CI systems. - --disabled-tests [pathToFile] - If specified, sets a path to a file that should have the names of disabled - tests on each row. - - --output-dir [pathToDir] - If specified, sets a directory to store the log files. This can be useful - for continuous integration. - --output-filename [filename] - If specified, sets a filename for the log files (within 'output-dir' or - (lacking that) the current directory. - By default, the name is constructed from the plugin metainformation - --verbose - If specified, outputs additional logging information. It can be useful to - turn this off when building with CI to avoid huge log files. - -Exit code: - 0 if all tests complete successfully - 1 if there are any errors - -Additionally, you can specify any of the command line options as environment -variables by removing prefix dashes, converting internal dashes to underscores -and capitalising all letters, a.g. - "--skip-gui-tests" > "SKIP_GUI_TESTS=1" - "--timeout-ms 30000" > "TIMEOUT_MS=30000" -Specifying specific command-line options will override any environment variables -set for that option. -)"); -} - -static juce::String getVersionText() -{ - return juce::String ("pluginval") + " - " + VERSION; -} - -static void printStrictnessHelp (int level) -{ - level = juce::jlimit (1, 10, level); - - std::cout << "Tests at strictness level " << level << ":\n\n"; - - auto& allTests = PluginTest::getAllTests(); - - std::vector sortedTests; - for (auto* test : allTests) - sortedTests.push_back (test); - - std::sort (sortedTests.begin(), sortedTests.end(), - [] (const PluginTest* a, const PluginTest* b) - { return a->strictnessLevel < b->strictnessLevel; }); - - for (auto* test : sortedTests) - { - if (test->strictnessLevel > level) - continue; - - auto descriptions = test->getDescription (level); - - for (const auto& desc : descriptions) - { - std::cout << " " << desc.title; - if (desc.description.isNotEmpty()) - std::cout << ": " << desc.description; - std::cout << "\n"; - } - } - - std::cout << std::endl; -} - -static int getNumTestFailures (juce::UnitTestRunner& testRunner) -{ - int numFailures = 0; - - for (int i = 0; i < testRunner.getNumResults(); ++i) - if (auto result = testRunner.getResult (i)) - numFailures += result->failures; - - return numFailures; -} - -static void runUnitTests() -{ - juce::UnitTestRunner testRunner; - testRunner.runTestsInCategory ("pluginval"); - const int numFailures = getNumTestFailures (testRunner); - - if (numFailures > 0) - juce::ConsoleApplication::fail (juce::String (numFailures) + " tests failed!!!"); -} - -//============================================================================== -static juce::ArgumentList createCommandLineArgs (juce::String commandLine) -{ - if (commandLine.contains ("strictnessLevel")) - { - std::cout << "!!! WARNING:\n\t\"strictnessLevel\" is deprecated and will be removed in a future version.\n" - << "\tPlease use --strictness-level instead\n\n"; - } - - commandLine = commandLine.replace ("strictnessLevel", "strictness-level") - .replace ("-NSDocumentRevisionsDebugMode YES", "") - .trim(); - - const auto exe = juce::File::getSpecialLocation (juce::File::currentExecutableFile); - - juce::StringArray args; - args.addTokens (commandLine, true); - args = mergeEnvironmentVariables (args); - args.trim(); - - for (auto& s : args) - s = s.unquoted(); - - // If only a plugin path is supplied as the last arg, add an implicit --validate - // option for it so the rest of the CLI works - juce::ArgumentList argList (exe.getFullPathName(), args); - - if (argList.size() > 0) - { - const bool hasValidateOrOtherCommand = argList.containsOption ("--validate") - || argList.containsOption ("--help|-h") - || argList.containsOption ("--version") - || argList.containsOption ("--run-tests"); - - if (! hasValidateOrOtherCommand) - if (isPluginArgument (argList.arguments.getLast().text)) - argList.arguments.insert (argList.arguments.size() - 1, { "--validate" }); - } - - return argList; -} - -static void performCommandLine (CommandLineValidator& validator, const juce::ArgumentList& args) -{ - hideDockIcon(); - - juce::ConsoleApplication cli; - cli.addVersionCommand ("--version", getVersionText()); - cli.addHelpCommand ("--help|-h", getHelpMessage(), true); - cli.addCommand ({ "--validate", - "--validate [pathToPlugin]", - "Validates the file (or IDs for AUs).", juce::String(), - [&validator] (const auto& validatorArgs) - { - auto [fileOrIDToValidate, options] = parseCommandLine (validatorArgs); - validator.validate (fileOrIDToValidate, options); - }}); - cli.addCommand ({ "--run-tests", - "--run-tests", - "Runs the internal unit tests.", juce::String(), - [] (const auto&) { runUnitTests(); }}); - cli.addCommand ({ "--strictness-help", - "--strictness-help [level]", - "Lists all tests that run at the given strictness level.", juce::String(), - [] (const auto& commandArgs) - { - int level = 5; - auto arg = getArgumentAfterOption (commandArgs, "--strictness-help"); - if (arg.text.isNotEmpty() && ! arg.isShortOption() && ! arg.isLongOption()) - level = arg.text.getIntValue(); - printStrictnessHelp (level); - }}); - - if (const auto retValue = cli.findAndRunCommand (args); retValue != 0) - { - juce::JUCEApplication::getInstance()->setApplicationReturnValue (retValue); - juce::JUCEApplication::getInstance()->quit(); - } - - // --validate runs async so will quit itself when done - if (! args.containsOption ("--validate")) - juce::JUCEApplication::getInstance()->quit(); -} - -//============================================================================== -void performCommandLine (CommandLineValidator& validator, const juce::String& commandLine) -{ - performCommandLine (validator, createCommandLineArgs (commandLine)); -} - -bool shouldPerformCommandLine (const juce::String& commandLine) -{ - const auto args = createCommandLineArgs (commandLine); - return args.containsOption ("--help|-h") - || args.containsOption ("--version") - || args.containsOption ("--validate") - || args.containsOption ("--run-tests") - || args.containsOption ("--strictness-help"); -} - -//============================================================================== -//============================================================================== -std::pair parseCommandLine (const juce::ArgumentList& args) -{ - auto fileOrID = getOptionValue (args, "--validate", "", "Expected a plugin path for the --validate option").toString(); - - // in the case of a path (vs. ID), grab the full path - // getCurrentWorkingDirectory is needed to handle relative paths - // It preserves absolute paths and first checks for ~ on Mac/Windows - if (fileOrID.contains ("~") || fileOrID.contains (".")) - fileOrID = juce::File::getCurrentWorkingDirectory().getChildFile(fileOrID).getFullPathName(); - - PluginTests::Options options; - options.strictnessLevel = getStrictnessLevel (args); - options.randomSeed = getRandomSeed (args); - options.timeoutMs = getTimeout (args); - options.verbose = args.containsOption ("--verbose"); - options.numRepeats = getNumRepeats (args); - options.randomiseTestOrder = args.containsOption ("--randomise"); - options.dataFile = getDataFile (args); - options.outputDir = getOutputDir (args); - options.outputFilename = getOutputFilename (args); - options.withGUI = ! args.containsOption ("--skip-gui-tests"); - options.disabledTests = getDisabledTests (args); - options.sampleRates = getSampleRates (args); - options.blockSizes = getBlockSizes (args); - options.realtimeCheck = magic_enum::enum_cast (getOptionValue (args, "--rtcheck", "", "Expected one of [disabled, enabled, relaxed]").toString().toStdString()) - .value_or (RealtimeCheck::disabled); - - return { fileOrID, options }; -} - -std::pair parseCommandLine (const juce::String& cmd) -{ - return parseCommandLine (createCommandLineArgs (cmd)); -} - -juce::StringArray createCommandLine (juce::String fileOrID, PluginTests::Options options) -{ - juce::StringArray args (juce::File::getSpecialLocation (juce::File::currentExecutableFile).getFullPathName()); - const PluginTests::Options defaults; - - if (options.strictnessLevel != defaults.strictnessLevel) - args.addArray ({ "--strictness-level", juce::String (options.strictnessLevel) }); - - if (options.randomSeed != defaults.randomSeed) - args.addArray ({ "--random-seed", juce::String (options.randomSeed) }); - - if (options.timeoutMs != defaults.timeoutMs) - args.addArray ({ "--timeout-ms", juce::String (options.timeoutMs) }); - - if (options.verbose) - args.add ("--verbose"); - - if (! options.withGUI) - args.add ("--skip-gui-tests"); - - if (options.numRepeats != defaults.numRepeats) - args.addArray ({ "--repeat", juce::String (options.numRepeats) }); - - if (options.randomiseTestOrder) - args.add ("--randomise"); - - if (options.dataFile != defaults.dataFile) - args.addArray ({ "--data-file", options.dataFile.getFullPathName() }); - - if (options.outputDir != defaults.outputDir) - args.addArray ({ "--output-dir", options.outputDir.getFullPathName() }); - - if (options.outputFilename != defaults.outputFilename) - args.addArray ({ "--output-filename", options.outputFilename }); - - if (options.disabledTests != defaults.disabledTests) - args.addArray ({ "--disabled-tests", options.disabledTests.joinIntoString (",") }); - - if (! options.sampleRates.empty()) - { - juce::StringArray sampleRates; - - for (auto rate : options.sampleRates) - sampleRates.add (juce::String (rate)); - - args.addArray ({ "--sample-rates", sampleRates.joinIntoString (",") }); - } - - if (! options.blockSizes.empty()) - { - juce::StringArray blockSizes; - - for (auto size : options.blockSizes) - blockSizes.add (juce::String (size)); - - args.addArray ({ "--block-sizes", blockSizes.joinIntoString (",") }); - } - - if (auto rtCheckMode = options.realtimeCheck; - rtCheckMode != RealtimeCheck::disabled) - { - args.addArray ({ "--rtcheck", std::string (magic_enum::enum_name (rtCheckMode)) }); - } - - args.addArray ({ "--validate", fileOrID }); - - return args; -} - -//============================================================================== -//============================================================================== -#include "CommandLineTests.cpp" diff --git a/Source/CommandLineTests.cpp b/Source/CommandLineTests.cpp deleted file mode 100644 index 9be45214..00000000 --- a/Source/CommandLineTests.cpp +++ /dev/null @@ -1,167 +0,0 @@ -/*============================================================================== - - Copyright 2018 by Tracktion Corporation. - For more information visit www.tracktion.com - - You may also use this code under the terms of the GPL v3 (see - www.gnu.org/licenses). - - pluginval IS PROVIDED "AS IS" WITHOUT ANY WARRANTY, AND ALL WARRANTIES, WHETHER - EXPRESSED OR IMPLIED, INCLUDING MERCHANTABILITY AND FITNESS FOR PURPOSE, ARE - DISCLAIMED. - - ==============================================================================*/ - - -struct CommandLineTests : public juce::UnitTest -{ - CommandLineTests() - : juce::UnitTest ("CommandLineTests", "pluginval") - { - } - - void runTest() override - { - - beginTest ("Merge environment variables"); - { - juce::StringPairArray envVars; - envVars.set ("STRICTNESS_LEVEL", "5"); - envVars.set ("RANDOM_SEED", "1234"); - envVars.set ("TIMEOUT_MS", "30000"); - envVars.set ("VERBOSE", "1"); - envVars.set ("REPEAT", "10"); - envVars.set ("RANDOMISE", "1"); - envVars.set ("SKIP_GUI_TESTS", "1"); - envVars.set ("DATA_FILE", ""); - envVars.set ("OUTPUT_DIR", ""); - - const auto merged = mergeEnvironmentVariables (juce::String(), [&envVars] (const juce::String& n, const juce::String& def) { return envVars.getValue (n, def); }).joinIntoString (" "); - expect (merged.contains ("--strictness-level 5")); - expect (merged.contains ("--random-seed 1234")); - expect (merged.contains ("--timeout-ms 30000")); - expect (merged.contains ("--verbose")); - expect (merged.contains ("--repeat 10")); - expect (merged.contains ("--randomise")); - expect (merged.contains ("--skip-gui-tests")); - expect (merged.contains ("--data-file ")); - expect (merged.contains ("--output-dir ")); - } - - beginTest ("Command line defaults"); - { - juce::ArgumentList args ({}, ""); - expectEquals (getStrictnessLevel (args), 5); - expectEquals (getRandomSeed (args), (juce::int64) 0); - expectEquals (getTimeout (args), (juce::int64) 30000); - expectEquals (getNumRepeats (args), 1); - expectEquals (getOptionValue (args, "--data-file", {}, "Missing data-file path argument!").toString(), juce::String()); - expectEquals (getOptionValue (args, "--output-dir", {}, "Missing output-dir path argument!").toString(), juce::String()); - } - - beginTest ("Command line parser"); - { - juce::ArgumentList args ({}, "--strictness-level 7 --random-seed 1234 --timeout-ms 20000 --repeat 11 --data-file /path/to/file --output-dir /path/to/dir --validate /path/to/plugin"); - expectEquals (getStrictnessLevel (args), 7); - expectEquals (getRandomSeed (args), (juce::int64) 1234); - expectEquals (getTimeout (args), (juce::int64) 20000); - expectEquals (getNumRepeats (args), 11); - expectEquals (getOptionValue (args, "--data-file", {}, "Missing data-file path argument!").toString(),juce::String ("/path/to/file")); - expectEquals (getOptionValue (args, "--output-dir", {}, "Missing output-dir path argument!").toString(),juce::String ("/path/to/dir")); - expectEquals (getOptionValue (args, "--validate", {}, "Missing validate argument!").toString(),juce::String ("/path/to/plugin")); - } - - beginTest ("Handles an absolute path to the plugin"); - { - const auto homeDir = juce::File::getSpecialLocation (juce::File::userHomeDirectory).getFullPathName(); - const auto commandLineString = "--validate " + homeDir + "/path/to/MyPlugin"; - const auto args = createCommandLineArgs (commandLineString); - expectEquals (parseCommandLine (args).first, homeDir + "/path/to/MyPlugin"); - } - - beginTest ("Handles a quoted absolute path to the plugin"); - { - const auto homeDir = juce::File::getSpecialLocation (juce::File::userHomeDirectory).getFullPathName(); - const auto pathToQuote = homeDir + "/path/to/MyPlugin"; - const auto commandLineString = "--validate " + pathToQuote.quoted(); - const auto args = createCommandLineArgs (commandLineString); - expectEquals (parseCommandLine (args).first, homeDir + "/path/to/MyPlugin"); - } - - beginTest ("Handles a relative path"); - { - const auto currentDir = juce::File::getCurrentWorkingDirectory(); - const auto args = createCommandLineArgs ("--validate MyPlugin.vst3"); - expectEquals (parseCommandLine (args).first, currentDir.getChildFile ("MyPlugin.vst3").getFullPathName()); - } - - beginTest ("Handles a quoted relative path with spaces to the plugin"); - { - const auto currentDir = juce::File::getCurrentWorkingDirectory(); - const auto args = createCommandLineArgs (R"(--validate "My Plugin.vst3")"); - expectEquals (parseCommandLine (args).first, currentDir.getChildFile ("My Plugin.vst3").getFullPathName()); - } - - #if !JUCE_WINDOWS - - beginTest ("Handles a relative path with ./ to the plugin"); - { - const auto currentDir = juce::File::getCurrentWorkingDirectory().getFullPathName(); - const auto commandLineString = "--validate ./path/to/MyPlugin"; - const auto args = createCommandLineArgs(commandLineString); - expectEquals (parseCommandLine (args).first, currentDir + "/path/to/MyPlugin"); - } - - beginTest ("Handles a home directory relative path to the plugin"); - { - const auto commandLineString = "--validate ~/path/to/MyPlugin"; - const auto args = createCommandLineArgs(commandLineString); - expectEquals (parseCommandLine (args).first, juce::File::getSpecialLocation (juce::File::userHomeDirectory).getFullPathName() + "/path/to/MyPlugin"); - } - - beginTest ("Handles quoted strings, spaces, and home directory relative path to the plugin"); - { - const auto commandLineString = R"(--data-file "~/path/to/My File" --output-dir "~/path/to/My Directory" --validate "~/path/to/My Plugin")"; - const auto args = createCommandLineArgs(commandLineString); - expectEquals (parseCommandLine (args).first, juce::File::getSpecialLocation (juce::File::userHomeDirectory).getFullPathName() + "/path/to/My Plugin"); - } - #endif - - beginTest ("Implicit validate with a relative path"); - { - const auto currentDir = juce::File::getCurrentWorkingDirectory(); - const auto args = createCommandLineArgs ("MyPlugin.vst3"); - expectEquals (parseCommandLine (args).first, currentDir.getChildFile ("MyPlugin.vst3").getFullPathName()); - } - - beginTest ("Doesn't alter component IDs"); - { - const auto commandLineString = "--validate MyPluginID"; - const auto args = createCommandLineArgs(commandLineString); - expectEquals (parseCommandLine (args).first,juce::String ("MyPluginID")); - } - - beginTest ("Command line random"); - { - expectEquals (getRandomSeed (juce::ArgumentList ({}, "--random-seed 0x7f2da1")), (juce::int64) 8334753); - expectEquals (getRandomSeed (juce::ArgumentList ({}, "--random-seed 0x692bc1f")), (juce::int64) 110279711); - } - - beginTest ("Implicit validate options"); - { - juce::TemporaryFile temp ("path_to_file.vst3"); - expect (temp.getFile().create()); - expect (shouldPerformCommandLine (temp.getFile().getFullPathName())); - } - - beginTest ("Allows for other options after explicit --validate"); - { - const auto currentDir = juce::File::getCurrentWorkingDirectory(); - const auto args = createCommandLineArgs ("--validate MyPlugin.vst3 --randomise"); - expectEquals (parseCommandLine (args).first, currentDir.getChildFile ("MyPlugin.vst3").getFullPathName()); - expect (parseCommandLine(args).second.randomiseTestOrder); - } - } -}; - -static CommandLineTests commandLineTests; diff --git a/Source/CrashHandler.cpp b/Source/CrashHandler.cpp deleted file mode 100644 index 50ba8cf9..00000000 --- a/Source/CrashHandler.cpp +++ /dev/null @@ -1,164 +0,0 @@ -/*============================================================================== - - Copyright 2018 by Tracktion Corporation. - For more information visit www.tracktion.com - - You may also use this code under the terms of the GPL v3 (see - www.gnu.org/licenses). - - pluginval IS PROVIDED "AS IS" WITHOUT ANY WARRANTY, AND ALL WARRANTIES, WHETHER - EXPRESSED OR IMPLIED, INCLUDING MERCHANTABILITY AND FITNESS FOR PURPOSE, ARE - DISCLAIMED. - - ==============================================================================*/ - -#include "juce_core/juce_core.h" -#include "CrashHandler.h" - -#if !JUCE_WINDOWS - #include - #include - #include - #include - #include - #include - #define CRASH_LOG "/tmp/pluginval_crash.txt" -#endif - -namespace -{ - juce::String getCrashLogContents() - { - return "\n" + juce::SystemStats::getStackBacktrace(); - } - - static juce::File getCrashTraceFile() - { - #if JUCE_WINDOWS - return juce::File::getSpecialLocation (juce::File::tempDirectory).getChildFile ("pluginval_crash.txt"); - #else - return juce::File(CRASH_LOG); - #endif - } - - #if JUCE_WINDOWS - static void handleCrash(void*) - { - const auto log = getCrashLogContents(); - std::cout << "\n*** FAILED: VALIDATION CRASHED\n" << log << std::endl; - getCrashTraceFile().replaceWithText (log); - } - #else - #ifdef __printflike - __printflike(2, 3) - #endif - static void writeToCrashLog(int fd, const char* fmt, ...) - { - char buf[1024]; - va_list args; - va_start(args, fmt); - // Warning: printf is not 100% async-signal-safe, but should be ok for locale-independent arguments like - // integers, strings, hex... floating point is locale-dependent and not safe to use here. - vsnprintf(buf, sizeof(buf), fmt, args); - va_end(args); - auto len = strlen(buf); - write(STDERR_FILENO, buf, len); - if (fd != -1) - write(fd, buf, len); - } - - static void handleCrash(void*) - { - // On Linux & Mac this is a signal handler, and therefore only "async-signal-safe" functions should be used. - // This means nothing that uses malloc (juce::File, juce::String, std::string, std::vector etc.) or buffered I/O. - - int fd = open(CRASH_LOG, O_RDWR | O_CREAT | O_TRUNC); - - const char *message = "\n*** FAILED: VALIDATION CRASHED\n"; - write(STDERR_FILENO, message, strlen(message)); - - // Recreate the output of backtrace_symbols(), which cannot be used in a signal handler because it uses malloc - static const int kMaxStacks = 128; - void *stacktrace[kMaxStacks] {}; - int stackCount = backtrace(stacktrace, kMaxStacks); - - static const int kMaxImages = 64; - const void *imageAddresses[kMaxImages] {}; - const char *imageNames[kMaxImages] {}; - int imageCount = 0; - - int skip = 2; // Skip handleCrash and juce::handleCrash) - for (int i = skip; i < stackCount; i++) - { - Dl_info info {}; - // Warning: dladdr can deadlock under rare conditions on macOS - if dyld is adding an image to its list - if (!dladdr(stacktrace[i], &info)) - { - writeToCrashLog(fd, "%-3d %-35s %p\n", i - skip, "", stacktrace[i]); - continue; - } - - const char *imageName = info.dli_fname ? strrchr(info.dli_fname, '/') : nullptr; - if (imageName) - { - imageName++; - - auto it = std::find(std::begin(imageAddresses), std::end(imageAddresses), info.dli_fbase); - if (it == std::end(imageAddresses) && imageCount < kMaxImages) - { - imageAddresses[imageCount] = info.dli_fbase; - imageNames[imageCount] = imageName; - imageCount++; - } - } - - if (info.dli_saddr) - { - ptrdiff_t offset = (char *)stacktrace[i] - (char *)info.dli_saddr; - writeToCrashLog(fd, "%-3d %-35s %p %s + %ld\n", i - skip, imageName, stacktrace[i], info.dli_sname, offset); - } - else - { - writeToCrashLog(fd, "%-3d %-35s %p\n", i - skip, imageName, stacktrace[i]); - } - } - - if (imageCount) - { - writeToCrashLog(fd, "\nBinary Images:"); - for (int i = 0; i < imageCount; i++) - writeToCrashLog(fd, "\n%p %s", imageAddresses[i], imageNames[i]); - writeToCrashLog(fd, "\n"); - } - - if (fd != -1) - close(fd); - - // Terminate normally to work around a bug in juce::ChildProcess::ActiveProcess::getExitStatus() - // which returns 0 (a "pass" in the host process) if the child process terminates abnormally. - // - https://github.com/Tracktion/pluginval/issues/125 - // - https://forum.juce.com/t/killed-childprocess-activeprocess-exit-code/61645/3 - // Use _Exit() instead of exit() so that static destructors don't run (they may not be async-signal-safe). - // FIXME: exiting here prevents Apple's Crash Reporter from creating reports. - std::_Exit(SIGKILL); - } - #endif -} - -//============================================================================== -void initialiseCrashHandler() -{ - // Delete the crash file, this will be created if possible - getCrashTraceFile().deleteFile(); - juce::SystemStats::setApplicationCrashHandler (handleCrash); -} - -juce::String getCrashLog() -{ - const auto f = getCrashTraceFile(); - - if (f.existsAsFile()) - return f.loadFileAsString(); - - return getCrashLogContents(); -} diff --git a/VERSION b/VERSION index ee90284c..227cea21 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.0.4 +2.0.0 diff --git a/docs/Acceptance testing.md b/docs/Acceptance testing.md new file mode 100644 index 00000000..6352c71d --- /dev/null +++ b/docs/Acceptance testing.md @@ -0,0 +1,106 @@ +# Acceptance testing + +Acceptance testing answers a different question from normal validation. Instead of +"does this plugin conform to the host API and behave safely?" it asks **"does this +plugin still produce the output I expect for a known input and state?"** - a +deterministic *render + golden-file comparison*. It is ideal for catching +accidental DSP changes (a wrong coefficient, a refactor that shifts the output) in +your own plugins from CI. + +It is driven by a small JSON config and a dedicated command: + +``` +pluginval test myPlugin-default.json +``` + +- The **first** run, with `--record-missing`, renders the plugin and, finding no + reference, **records** one (a `.wav` plus a `.wav.json` manifest) next to the + config and reports success. Without `--record-missing` a missing reference is an + error, so a golden file that is deleted or mis-pathed can't silently pass in CI. +- **Subsequent** runs render again and **compare** against that reference, + exiting `0` on a match and `1` on a mismatch (writing a diff `.wav` to help you + see what changed). + +So the workflow is: write a config, run once with `--record-missing` to record the +reference, **commit the +config and the reference**, then let CI run the same command on every change. + +### A minimal config + +```json +{ + "name": "myReverb-default", + "plugin": "/path/to/MyReverb.vst3", + "input": { "audio": "inputs/drums.wav" }, + "reference": "refs/myReverb-default.wav", + "state": { "parameters": { "Mix": 0.5, "Decay": 0.8 } }, + "sample_rate": 48000, + "block_size": 512, + "render_duration": 2.0, + "comparison": { "sample": 1e-6 } +} +``` + +JSON keys are `snake_case`. The most useful fields: + +| Field | Notes | +|---|---| +| `name` | Labels the result and gives the default reference path. **Required** unless `reference` is given. | +| `plugin` | Path to the plugin (or an AU identifier). **Required.** | +| `input.audio` / `input.midi` | Input files to feed it. Omit both for silence (e.g. instruments driven only by MIDI, or generators). The audio file's sample rate must match `sample_rate`. Each file channel feeds the plugin input channel with the same index: a mono file feeds channel 0 only (it isn't duplicated), and extra channels on either side are silent / ignored. MIDI meta events (tempo, track names) aren't sent to the plugin. | +| `state.parameters` | A map of parameter → **normalised** value (`0` to `1`), applied before rendering, **in file order**. A key of digits only is a parameter index; otherwise it matches the format's parameter ID, then the display name (case-insensitively). | +| `state.file` | A binary `getStateInformation` blob to restore first (e.g. a captured preset). Applied before `state.parameters`. | +| `reference` | The golden `.wav`. Defaults to `.wav` next to the config. | +| `sample_rate` / `block_size` | Default `44100` / `512`. `sample_rate` must be positive; `block_size` must be 1 to 65536. | +| `render_duration` | Seconds to render (positive). If omitted, the input audio's length is used. | +| `playhead` | A fixed transport for tempo-dependent plugins: `{ "bpm": 120, "time_signature": { "numerator": 4, "denominator": 4 } }`. Omit it and the plugin gets no playhead. The position advances with the render. | +| `comparison` | How to compare. `{ "sample": }` is a per-sample absolute-difference tolerance (`0` = bit-exact); the default is one 16-bit LSB. Any NaN / inf difference fails. Omit it for the default; an empty `{}` is an error. | + +An invalid config (a missing required field or an out-of-range value) is reported +as an error (exit `1`) without rendering. + +### Determinism matters + +Acceptance testing only works for output that is reproducible from a fixed input +and state. A plugin with free-running randomness can't be golden-tested reliably. +For the same reason, references are only safely **portable across platforms** when +you allow a tolerance - exact per-sample matches rarely survive different CPUs and +floating-point libraries. If a reference recorded on one OS fails on another, raise +the `sample` tolerance (or use a more tolerant comparison method as they are added). + +### Running from CI + +`pluginval test` is just a command that returns an exit code, so any CI system can +run it the same way it runs your other checks: + +```bash +pluginval test tests/acceptance/myReverb-default.json +``` + +A non-zero exit fails the build. Commit the config and its reference `.wav` +alongside your project so every run compares against the same golden file. + +Options: + +Any other option is an error. + +- `--record-missing` - record a missing reference and pass instead of failing. + Useful if your CI records new references and checks them in. Also settable as + `RECORD_MISSING=1`. +- `--timeout-ms ` - fail (exit `1`) if the run takes longer than this + (default `30000`, `-1` to never time out). Also settable as `TIMEOUT_MS`. + +The render runs on a background thread, as a host's audio thread would, leaving +the message thread free for the plugin. + +Each reference's `.wav.json` manifest stores a `config_hash`. If the config (or the +contents of its input / state files) changes after the reference was recorded, the +run warns that the reference may be stale. Paths and the plugin binary aren't part +of the hash, so a checked-in reference doesn't warn on other machines. + +On a mismatch a `-diff.wav` (output minus reference) is written next to the +reference; it's removed again on the next passing run. Add `*-diff.wav` to your +`.gitignore`. + +For the complete schema, the comparator design and the planned roadmap, see the +[design document](<../tests/acceptance/Acceptance testing design.md>). diff --git a/docs/Adding pluginval to CI.md b/docs/Adding pluginval to CI.md index e7e96de7..3baa06b9 100644 --- a/docs/Adding pluginval to CI.md +++ b/docs/Adding pluginval to CI.md @@ -19,21 +19,21 @@ In these examples the following is assumed: ```sh $ curl -L "https://github.com/Tracktion/pluginval/releases/latest/download/pluginval_macOS.zip" -o pluginval.zip $ unzip pluginval -$ pluginval.app/Contents/MacOS/pluginval --validate-in-process --output-dir "./bin" "" || exit 1 +$ pluginval.app/Contents/MacOS/pluginval validate --output-dir "./bin" "" || exit 1 ``` ##### Linux ```sh $ curl -L "https://github.com/Tracktion/pluginval/releases/latest/download/pluginval_Linux.zip" -o pluginval.zip $ unzip pluginval -$ ./pluginval --validate-in-process --output-dir "./bin" "" || exit 1 +$ ./pluginval validate --output-dir "./bin" "" || exit 1 ``` ##### Windows ```sh > powershell -Command "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; Invoke-WebRequest https://github.com/Tracktion/pluginval/releases/latest/download/pluginval_Windows.zip -OutFile pluginval.zip" > powershell -Command "Expand-Archive pluginval.zip -DestinationPath ." -> pluginval.exe --validate-in-process --output-dir "./bin" "" +> pluginval.exe validate --output-dir "./bin" "" > if %ERRORLEVEL% neq 0 exit /b 1 ``` diff --git a/docs/Command line options.md b/docs/Command line options.md index 4af7339f..4d1bc568 100644 --- a/docs/Command line options.md +++ b/docs/Command line options.md @@ -1,85 +1,65 @@ -Hello rtcheck! -//============================================================================== -pluginval -JUCE v8.0.10 +Validate plugins to test compatibility with hosts and verify plugin API +conformance -Description: - Validate plugins to test compatibility with hosts and verify plugin API conformance -Usage: - --version - Print pluginval version. - --validate [pathToPlugin] - Validates the plugin at the given path. - N.B. the "--validate" flag is optional if the path is the last argument. - This enables you to validate a plugin with simply "pluginval path_to_plugin". +pluginval [OPTIONS] [plugin] - --sample-rates [list of comma separated sample rates] - If specified, sets the list of sample rates at which tests will be executed - (default=44100,48000,96000) - --block-sizes [list of comma separated block sizes] - If specified, sets the list of block sizes at which tests will be executed - (default=64,128,256,512,1024) - --random-seed [hex or int] - Sets the random seed to use for the tests. Useful for replicating test - environments. - --data-file [pathToFile] - If specified, sets a path to a data file which can be used by tests to - configure themselves. This can be useful for things like known audio output. - --strictness-level [1-10] - Sets the strictness level to use. A minimum level of 5 (also the default) - is recomended for compatibility. - Higher levels include longer, more thorough tests such as fuzzing. - --timeout-ms [numMilliseconds] - Sets a timout which will stop validation with an error if no output from any - test has happened for this number of ms. - By default this is 30s but can be set to "-1" (must be quoted) to never timeout. - --rtcheck [empty, disabled, enabled or relaxed] - Turns on real-time saftey checks using rtcheck (macOS and Linux only). - relaxed mode doesn't run the checks for the first processing block as a lot of plugins - use this to allocate or initialise thread-locals (which can allocate) +COMMANDS: + validate [options] Validate the plugin at the given path or AU id. + This is the default, so "pluginval " and + "pluginval [options] " also work. + run-tests Run the internal unit tests. + strictness-help [level] List all tests that run at the given strictness level. + test [options] Run a deterministic acceptance (golden-file) test + from a config. Compares against the reference + (exit 0/1); a missing reference is an error. + --record-missing Record a missing reference and + pass instead. + --timeout-ms Fail if the run takes longer + (default 30000, -1 = never). + Any other option is an error. - --repeat [num repeats] - If specified repeats the tests a given number of times. Note that this does - not delete and re-instantiate the plugin for each repeat. - --randomise - If specified, the tests are run in a random order per repeat. +The flat flags --validate , --run-tests and --strictness-help [level] +are deprecated aliases for the commands above and will be removed in a future +version. - --skip-gui-tests - If specified, avoids tests that create GUI windows, which can cause problems - on headless CI systems. - --disabled-tests [pathToFile] - If specified, sets a path to a file that should have the names of disabled - tests on each row. - --vst3validator [pathToValidator] - If specified, this will run the VST3 validator as part of the test process. - --output-dir [pathToDir] - If specified, sets a directory to store the log files. This can be useful - for continuous integration. - --output-filename [filename] - If specified, sets a filename for the log files (within 'output-dir' or - (lacking that) the current directory. - By default, the name is constructed from the plugin metainformation - --verbose - If specified, outputs additional logging information. It can be useful to - turn this off when building with CI to avoid huge log files. +OPTIONS (for the validate command): + -h, --help Print this help message and exit + --version Display program version information and exit + --config TEXT ... Path to a JSON settings file. Repeatable; later files win per + key. + --validate TEXT Validates the plugin at the given path (or AU id). + --strictness-level INT + Strictness level 1-10 (default 5). + --timeout-ms INT Test timeout in ms (default 30000, -1 to never timeout). + --repeat INT Number of times to repeat the tests. + --randomise Run the tests in a random order per repeat. + --verbose Output additional logging information. + --skip-gui-tests Avoid tests that create GUI windows (for headless CI). + --sample-rates FLOAT ... + Comma-separated sample rates (default 44100,48000,96000). + --block-sizes INT ... + Comma-separated block sizes (default 64,128,256,512,1024). + --data-file TEXT Path to a data file tests can use to configure themselves. + --output-dir TEXT Directory in which to write the log files. + --output-filename TEXT + Filename to write logs into. + --disabled-tests TEXT + Comma-separated test names, or a path to a file listing them. + --random-seed TEXT Random seed (hex 0x.. or int) for replicable test runs. + --rtcheck ENUM:value in {disabled->0,enabled->1,relaxed->2} OR {0,1,2} + Real-time safety checks: disabled, enabled or relaxed. -Exit code: - 0 if all tests complete successfully - 1 if there are any errors - -Additionally, you can specify any of the command line options as environment -variables by removing prefix dashes, converting internal dashes to underscores -and capitalising all letters, a.g. - "--skip-gui-tests" > "SKIP_GUI_TESTS=1" - "--timeout-ms 30000" > "TIMEOUT_MS=30000" -Specifying specific command-line options will override any environment variables -set for that option. - - pluginval --version Prints the current version number - pluginval --help|-h Prints the list of commands - pluginval --validate [pathToPlugin] Validates the file (or IDs for AUs). - pluginval --run-tests Runs the internal unit tests. +Exit code: +0 if all tests complete successfully +1 if there are any errors +You can also specify any option as an environment variable by removing the +prefix dashes, converting internal dashes to underscores and capitalising, e.g. +"--skip-gui-tests" -> "SKIP_GUI_TESTS=1" +"--timeout-ms 30000" -> "TIMEOUT_MS=30000" +Precedence (lowest to highest): defaults, environment variables, --config, +command-line options. +--config is repeatable; later files win per key. diff --git a/source/CommandLine.cpp b/source/CommandLine.cpp new file mode 100644 index 00000000..f317557b --- /dev/null +++ b/source/CommandLine.cpp @@ -0,0 +1,280 @@ +/*============================================================================== + + Copyright 2018 by Tracktion Corporation. + For more information visit www.tracktion.com + + You may also use this code under the terms of the GPL v3 (see + www.gnu.org/licenses). + + pluginval IS PROVIDED "AS IS" WITHOUT ANY WARRANTY, AND ALL WARRANTIES, WHETHER + EXPRESSED OR IMPLIED, INCLUDING MERCHANTABILITY AND FITNESS FOR PURPOSE, ARE + DISCLAIMED. + + ==============================================================================*/ + +#include "CommandLine.h" +#include "Validator.h" +#include "CrashHandler.h" +#include "PluginTests.h" +#include "PluginvalSettings.h" +#include "SettingsParser.h" +#include "acceptance/AcceptanceTest.h" + +#include +#include + +#if JUCE_MAC + #include + #include +#endif + +//============================================================================== +static void exitWithError (const juce::String& error) +{ + std::cout << error << std::endl << std::endl; + juce::JUCEApplication::getInstance()->setApplicationReturnValue (1); + juce::JUCEApplication::getInstance()->quit(); +} + +static void hideDockIcon() +{ + #if JUCE_MAC + juce::Process::setDockIconVisible (false); + #endif +} + +inline std::mutex& getCoutMutex() +{ + static std::mutex m; + return m; +} + +inline void logLine (const juce::String& m) +{ + const std::scoped_lock sl (getCoutMutex()); + std::cout << m << std::endl; +} + +inline void logAndFlush (const juce::String& m) +{ + const std::scoped_lock sl (getCoutMutex()); + std::cout << m << std::flush; +} + +//============================================================================== +//============================================================================== +void CommandLineValidator::validate (const juce::String& fileOrID, PluginTests::Options options) +{ + validator = std::make_unique (fileOrID, options, ValidationType::inProcess, + [] (auto id) + { + logLine ("Started validating: " + id); + }, + [] (auto, uint32_t exitCode) + { + if (exitCode > 0) + exitWithError ("*** FAILED"); + else + juce::JUCEApplication::getInstance()->quit(); + }, + [] (auto m) + { + logAndFlush (m); + }); +} + +void CommandLineValidator::runAcceptanceTest (const juce::File& configFile, const acceptance::RunOptions& options) +{ + // Runs on a background thread so the message thread stays free for the + // plugin; quits itself when done. + testRunner = std::make_unique (configFile, options, + [] (int exitCode) + { + auto& app = *juce::JUCEApplication::getInstance(); + app.setApplicationReturnValue (exitCode); + app.quit(); + }); +} + +//============================================================================== +static void printStrictnessHelp (int level) +{ + level = juce::jlimit (1, 10, level); + + std::cout << "Tests at strictness level " << level << ":\n\n"; + + auto& allTests = PluginTest::getAllTests(); + + std::vector sortedTests; + for (auto* test : allTests) + sortedTests.push_back (test); + + std::sort (sortedTests.begin(), sortedTests.end(), + [] (const PluginTest* a, const PluginTest* b) + { return a->strictnessLevel < b->strictnessLevel; }); + + for (auto* test : sortedTests) + { + if (test->strictnessLevel > level) + continue; + + auto descriptions = test->getDescription (level); + + for (const auto& desc : descriptions) + { + std::cout << " " << desc.title; + if (desc.description.isNotEmpty()) + std::cout << ": " << desc.description; + std::cout << "\n"; + } + } + + std::cout << std::endl; +} + +static void runUnitTests() +{ + juce::UnitTestRunner testRunner; + testRunner.runTestsInCategory ("pluginval"); + + int numFailures = 0; + + // Print failures to stdout: juce::UnitTestRunner logs via juce::Logger, which + // on a GUI app (e.g. Windows) doesn't reach the console. + for (int i = 0; i < testRunner.getNumResults(); ++i) + { + if (auto* result = testRunner.getResult (i)) + { + numFailures += result->failures; + + if (result->failures > 0) + { + std::cout << "!!! FAILED: " << result->unitTestName << " / " << result->subcategoryName << std::endl; + + for (const auto& message : result->messages) + std::cout << " " << message << std::endl; + } + } + } + + // Set the return value directly rather than juce::ConsoleApplication::fail(), + // which throws and would terminate the process when called outside a + // ConsoleApplication command handler. + if (numFailures > 0) + { + std::cout << numFailures << " tests failed!!!" << std::endl; + juce::JUCEApplication::getInstance()->setApplicationReturnValue (1); + } +} + +//============================================================================== +//============================================================================== +std::pair parseCommandLine (const juce::String& commandLine) +{ + const auto settings = settings_parser::parse (commandLine); + return { juce::String (settings.validatePath), settings.toPluginTestOptions() }; +} + +juce::StringArray createCommandLine (juce::String fileOrID, PluginTests::Options options) +{ + return settings_parser::createChildProcessCommandLine (fileOrID, options); +} + +//============================================================================== +static void warnDeprecated (const juce::String& oldForm, const juce::String& newForm) +{ + std::cerr << "!!! WARNING: " << oldForm << " is deprecated; use '" << newForm + << "' instead. It will be removed in a future version." << std::endl; +} + +void performCommandLine (CommandLineValidator& validator, const juce::String& commandLine) +{ + hideDockIcon(); + + auto& app = *juce::JUCEApplication::getInstance(); + const auto routed = settings_parser::dispatch (settings_parser::tokenise (commandLine)); + + if (routed.command == settings_parser::Command::runTests) + { + if (routed.deprecatedAlias) + warnDeprecated ("--run-tests", "pluginval run-tests"); + + runUnitTests(); + app.quit(); + return; + } + + if (routed.command == settings_parser::Command::strictnessHelp) + { + if (routed.deprecatedAlias) + warnDeprecated ("--strictness-help", "pluginval strictness-help"); + + printStrictnessHelp (routed.strictnessLevel); + app.quit(); + return; + } + + if (routed.command == settings_parser::Command::test) + { + if (routed.error.isNotEmpty()) + { + exitWithError ("*** FAILED: " + routed.error); + return; + } + + if (routed.testConfigPath.isEmpty()) + { + exitWithError ("*** FAILED: No acceptance-test config specified (usage: pluginval test )"); + return; + } + + acceptance::RunOptions options; + options.recordMissing = routed.recordMissing; + options.timeoutMs = routed.timeoutMs; + + // Runs async so will quit itself when done + validator.runAcceptanceTest (juce::File::getCurrentWorkingDirectory().getChildFile (routed.testConfigPath), options); + return; + } + + // Otherwise this is a validation run (positional plugin, or explicit/implicit + // --validate). CLI11 handles --help/--version and parse errors. + if (routed.deprecatedAlias) + warnDeprecated ("--validate", "pluginval validate "); + + try + { + const auto result = settings_parser::parseTokens (routed.validateTokens); + + if (result.handled) + { + app.setApplicationReturnValue (result.exitCode); + app.quit(); + return; + } + + const auto fileOrID = juce::String (result.settings.validatePath); + + if (fileOrID.isEmpty()) + { + exitWithError ("*** FAILED: No plugin path or ID specified to validate"); + return; + } + + // --validate runs async so will quit itself when done + validator.validate (fileOrID, result.settings.toPluginTestOptions()); + } + catch (const std::exception& e) + { + exitWithError (juce::String ("*** FAILED: ") + e.what()); + } +} + +bool shouldPerformCommandLine (const juce::String& commandLine) +{ + return settings_parser::isCommandLine (settings_parser::preprocess (commandLine)); +} + +//============================================================================== +//============================================================================== +#include "CommandLineTests.cpp" diff --git a/Source/CommandLine.h b/source/CommandLine.h similarity index 76% rename from Source/CommandLine.h rename to source/CommandLine.h index 644eb55a..ce6d153b 100644 --- a/Source/CommandLine.h +++ b/source/CommandLine.h @@ -16,17 +16,20 @@ #include "juce_core/juce_core.h" #include "Validator.h" +#include "acceptance/AcceptanceTest.h" //============================================================================== struct CommandLineValidator { - CommandLineValidator(); - ~CommandLineValidator(); + CommandLineValidator() = default; + ~CommandLineValidator() = default; void validate (const juce::String&, PluginTests::Options); + void runAcceptanceTest (const juce::File& configFile, const acceptance::RunOptions&); private: std::unique_ptr validator; + std::unique_ptr testRunner; }; //============================================================================== @@ -34,6 +37,8 @@ void performCommandLine (CommandLineValidator&, const juce::String& commandLine) bool shouldPerformCommandLine (const juce::String& commandLine); //============================================================================== +/** Parses a command line into the plugin path/ID and resolved test options. */ std::pair parseCommandLine (const juce::String&); -std::pair parseCommandLine (const juce::ArgumentList&); + +/** Serialises options for the child validation process. */ juce::StringArray createCommandLine (juce::String fileOrID, PluginTests::Options); diff --git a/source/CommandLineTests.cpp b/source/CommandLineTests.cpp new file mode 100644 index 00000000..70b5d510 --- /dev/null +++ b/source/CommandLineTests.cpp @@ -0,0 +1,660 @@ +/*============================================================================== + + Copyright 2018 by Tracktion Corporation. + For more information visit www.tracktion.com + + You may also use this code under the terms of the GPL v3 (see + www.gnu.org/licenses). + + pluginval IS PROVIDED "AS IS" WITHOUT ANY WARRANTY, AND ALL WARRANTIES, WHETHER + EXPRESSED OR IMPLIED, INCLUDING MERCHANTABILITY AND FITNESS FOR PURPOSE, ARE + DISCLAIMED. + + ==============================================================================*/ + +#include +#include + +struct CommandLineTests : public juce::UnitTest +{ + CommandLineTests() + : juce::UnitTest ("CommandLineTests", "pluginval") + { + } + + static settings_parser::EnvProvider emptyEnv() + { + return [] (const juce::String&) { return juce::String(); }; + } + + static settings_parser::EnvProvider envFrom (std::map vars) + { + return [vars = std::move (vars)] (const juce::String& name) + { + const auto it = vars.find (name); + return it != vars.end() ? it->second : juce::String(); + }; + } + + static PluginvalSettings parse (const juce::String& cmd, settings_parser::EnvProvider env) + { + return settings_parser::parse (cmd, env); + } + + static PluginvalSettings parse (const juce::String& cmd) + { + return settings_parser::parse (cmd, emptyEnv()); + } + + void runTest() override + { + beginTest ("Command line defaults"); + { + const auto opts = parse ("").toPluginTestOptions(); + expectEquals (opts.strictnessLevel, 5); + expectEquals (opts.randomSeed, (juce::int64) 0); + expectEquals (opts.timeoutMs, (juce::int64) 30000); + expectEquals (opts.numRepeats, 1); + expect (opts.verbose == false); + expect (opts.randomiseTestOrder == false); + expect (opts.withGUI == true); + expect (opts.realtimeCheck == RealtimeCheck::disabled); + expect (opts.dataFile == juce::File()); + expect (opts.outputDir == juce::File()); + expect (opts.sampleRates == PluginvalSettings::defaultSampleRates()); + expect (opts.blockSizes == PluginvalSettings::defaultBlockSizes()); + } + + beginTest ("Command line parser"); + { + const auto settings = parse ("--strictness-level 7 --random-seed 1234 --timeout-ms 20000 --repeat 11 " + "--data-file /path/to/file --output-dir /path/to/dir --validate /path/to/plugin"); + const auto opts = settings.toPluginTestOptions(); + expectEquals (opts.strictnessLevel, 7); + expectEquals (opts.randomSeed, (juce::int64) 1234); + expectEquals (opts.timeoutMs, (juce::int64) 20000); + expectEquals (opts.numRepeats, 11); + // Compare the raw parsed strings: juce::File would normalise these to the + // current drive on Windows (e.g. "D:\path\to\file"). + expectEquals (juce::String (settings.dataFile), juce::String ("/path/to/file")); + expectEquals (juce::String (settings.outputDir), juce::String ("/path/to/dir")); + expectEquals (juce::String (settings.validatePath), juce::String ("/path/to/plugin")); + } + + beginTest ("Negative timeout"); + { + expectEquals ((juce::int64) parse ("--timeout-ms -1 --validate x").timeoutMs, (juce::int64) -1); + } + + beginTest ("Command line random (hex and int)"); + { + expectEquals ((juce::int64) parse ("--random-seed 0x7f2da1 --validate x").randomSeed, (juce::int64) 8334753); + expectEquals ((juce::int64) parse ("--random-seed 0x692bc1f --validate x").randomSeed, (juce::int64) 110279711); + expectEquals ((juce::int64) parse ("--random-seed 1234 --validate x").randomSeed, (juce::int64) 1234); + } + + beginTest ("Comma-separated lists"); + { + const auto opts = parse ("--sample-rates 22050,44100 --block-sizes 32,64,128 --validate x").toPluginTestOptions(); + expect (opts.sampleRates == std::vector ({ 22050.0, 44100.0 })); + expect (opts.blockSizes == std::vector ({ 32, 64, 128 })); + } + + beginTest ("rtcheck enum parsing"); + { + expect (parse ("--rtcheck relaxed --validate x").realtimeCheck == RealtimeCheck::relaxed); + expect (parse ("--rtcheck enabled --validate x").realtimeCheck == RealtimeCheck::enabled); + expect (parse ("--validate x").realtimeCheck == RealtimeCheck::disabled); + } + + beginTest ("Handles an absolute path to the plugin"); + { + const auto homeDir = juce::File::getSpecialLocation (juce::File::userHomeDirectory).getFullPathName(); + expectEquals (juce::String (parse ("--validate " + homeDir + "/path/to/MyPlugin").validatePath), + homeDir + "/path/to/MyPlugin"); + } + + beginTest ("Handles a quoted absolute path to the plugin"); + { + const auto homeDir = juce::File::getSpecialLocation (juce::File::userHomeDirectory).getFullPathName(); + const auto pathToQuote = homeDir + "/path/to/MyPlugin"; + expectEquals (juce::String (parse ("--validate " + pathToQuote.quoted()).validatePath), + homeDir + "/path/to/MyPlugin"); + } + + beginTest ("Handles a relative path"); + { + const auto currentDir = juce::File::getCurrentWorkingDirectory(); + expectEquals (juce::String (parse ("--validate MyPlugin.vst3").validatePath), + currentDir.getChildFile ("MyPlugin.vst3").getFullPathName()); + } + + beginTest ("Handles a quoted relative path with spaces to the plugin"); + { + const auto currentDir = juce::File::getCurrentWorkingDirectory(); + expectEquals (juce::String (parse (R"(--validate "My Plugin.vst3")").validatePath), + currentDir.getChildFile ("My Plugin.vst3").getFullPathName()); + } + + #if !JUCE_WINDOWS + beginTest ("Handles a relative path with ./ to the plugin"); + { + const auto currentDir = juce::File::getCurrentWorkingDirectory().getFullPathName(); + expectEquals (juce::String (parse ("--validate ./path/to/MyPlugin").validatePath), + currentDir + "/path/to/MyPlugin"); + } + + beginTest ("Handles a home directory relative path to the plugin"); + { + expectEquals (juce::String (parse ("--validate ~/path/to/MyPlugin").validatePath), + juce::File::getSpecialLocation (juce::File::userHomeDirectory).getFullPathName() + "/path/to/MyPlugin"); + } + + beginTest ("Handles quoted strings, spaces, and home directory relative path to the plugin"); + { + const auto cmd = R"(--data-file "~/path/to/My File" --output-dir "~/path/to/My Directory" --validate "~/path/to/My Plugin")"; + expectEquals (juce::String (parse (cmd).validatePath), + juce::File::getSpecialLocation (juce::File::userHomeDirectory).getFullPathName() + "/path/to/My Plugin"); + } + #endif + + beginTest ("Implicit validate with a relative path"); + { + const auto currentDir = juce::File::getCurrentWorkingDirectory(); + expectEquals (juce::String (parse ("MyPlugin.vst3").validatePath), + currentDir.getChildFile ("MyPlugin.vst3").getFullPathName()); + } + + beginTest ("Doesn't alter component IDs"); + { + expectEquals (juce::String (parse ("--validate MyPluginID").validatePath), juce::String ("MyPluginID")); + } + + beginTest ("Allows for other options after explicit --validate"); + { + const auto currentDir = juce::File::getCurrentWorkingDirectory(); + const auto settings = parse ("--validate MyPlugin.vst3 --randomise"); + expectEquals (juce::String (settings.validatePath), currentDir.getChildFile ("MyPlugin.vst3").getFullPathName()); + expect (settings.randomiseTestOrder); + } + + beginTest ("Should perform command line"); + { + juce::TemporaryFile temp ("path_to_file.vst3"); + expect (temp.getFile().create()); + expect (shouldPerformCommandLine (temp.getFile().getFullPathName())); + expect (shouldPerformCommandLine ("--run-tests")); + expect (shouldPerformCommandLine ("--version")); + expect (! shouldPerformCommandLine ("")); + } + + beginTest ("Environment variables"); + { + const auto env = envFrom ({ + { "STRICTNESS_LEVEL", "7" }, + { "RANDOM_SEED", "1234" }, + { "TIMEOUT_MS", "20000" }, + { "VERBOSE", "1" }, + { "REPEAT", "11" }, + { "RANDOMISE", "1" }, + { "SKIP_GUI_TESTS", "1" }, + { "DATA_FILE", "/path/to/file" }, + { "OUTPUT_DIR", "/path/to/dir" }, + { "SAMPLE_RATES", "22050,44100" }, + { "BLOCK_SIZES", "32,64" }, + { "RTCHECK", "relaxed" }, + }); + + const auto settings = parse ("--validate x", env); + const auto opts = settings.toPluginTestOptions(); + expectEquals (opts.strictnessLevel, 7); + expectEquals (opts.randomSeed, (juce::int64) 1234); + expectEquals (opts.timeoutMs, (juce::int64) 20000); + expect (opts.verbose); + expectEquals (opts.numRepeats, 11); + expect (opts.randomiseTestOrder); + expect (opts.withGUI == false); + expectEquals (juce::String (settings.dataFile), juce::String ("/path/to/file")); + expect (opts.sampleRates == std::vector ({ 22050.0, 44100.0 })); + expect (opts.blockSizes == std::vector ({ 32, 64 })); + expect (opts.realtimeCheck == RealtimeCheck::relaxed); + } + + beginTest ("Command line overrides environment variables"); + { + const auto env = envFrom ({ { "STRICTNESS_LEVEL", "3" } }); + expectEquals (parse ("--validate x", env).strictnessLevel, 3); // env only + expectEquals (parse ("--strictness-level 9 --validate x", env).strictnessLevel, 9); // CLI wins + } + + beginTest ("Precedence: CLI > --config > env > defaults"); + { + juce::TemporaryFile configFile (".json"); + configFile.getFile().replaceWithText (R"({ "strictnessLevel": 2, "timeoutMs": 12345 })"); + const auto cfg = "--config " + configFile.getFile().getFullPathName().quoted(); + + const auto env6 = envFrom ({ { "STRICTNESS_LEVEL", "6" } }); + + // env beats defaults + expectEquals (parse ("--validate x", env6).strictnessLevel, 6); + + // config alone + { + const auto s = parse (cfg + " --validate x"); + expectEquals (s.strictnessLevel, 2); + expectEquals ((juce::int64) s.timeoutMs, (juce::int64) 12345); + } + + // config beats env + { + const auto s = parse (cfg + " --validate x", env6); + expectEquals (s.strictnessLevel, 2); // config wins over env + expectEquals ((juce::int64) s.timeoutMs, (juce::int64) 12345); + } + + // CLI beats config and env + { + const auto s = parse (cfg + " --strictness-level 9 --validate x", env6); + expectEquals (s.strictnessLevel, 9); + expectEquals ((juce::int64) s.timeoutMs, (juce::int64) 12345); + } + } + + beginTest ("Repeatable --config merges per key, last wins"); + { + juce::TemporaryFile baseFile (".json"); + baseFile.getFile().replaceWithText (R"({ "strictnessLevel": 2, "timeoutMs": 11111 })"); + + juce::TemporaryFile overrideFile (".json"); + overrideFile.getFile().replaceWithText (R"({ "strictnessLevel": 8 })"); + + const auto cmd = "--config " + baseFile.getFile().getFullPathName().quoted() + + " --config " + overrideFile.getFile().getFullPathName().quoted() + + " --validate x"; + + const auto s = parse (cmd); + expectEquals (s.strictnessLevel, 8); // overridden by the second file + expectEquals ((juce::int64) s.timeoutMs, (juce::int64) 11111); // untouched, kept from the first + } + + beginTest ("Child-process command line round-trip"); + { + PluginTests::Options opts; + opts.strictnessLevel = 8; + opts.randomSeed = 8334753; + opts.timeoutMs = 15000; + opts.verbose = true; + opts.numRepeats = 3; + opts.randomiseTestOrder = true; + opts.withGUI = false; + opts.outputFilename = "log.txt"; + opts.disabledTests = juce::StringArray ({ "Test A", "Test B" }); + opts.sampleRates = { 44100.0, 96000.0 }; + opts.blockSizes = { 64, 512 }; + opts.realtimeCheck = RealtimeCheck::relaxed; + + const juce::String fileOrID = "/some/dir/MyPlugin.vst3"; + + const auto args = createCommandLine (fileOrID, opts); + + juce::StringArray childArgs (args); + childArgs.remove (0); // drop the executable path + const auto childCommandLine = childArgs.joinIntoString (" "); + + const auto [fileOrID2, opts2] = parseCommandLine (childCommandLine); + + auto expected = PluginvalSettings::fromPluginTestOptions (opts, fileOrID); + expected.validatePath = settings_parser::resolvePluginPath (fileOrID).toStdString(); + + expect (PluginvalSettings::fromPluginTestOptions (opts2, fileOrID2) == expected); + } + + beginTest ("--config-base64 rejects extra options"); + { + PluginTests::Options opts; + opts.strictnessLevel = 7; + + juce::StringArray childArgs (createCommandLine ("/some/MyPlugin.vst3", opts)); + childArgs.remove (0); // drop the executable path + + // The legitimate parent -> child handoff parses fine. + { + const auto r = settings_parser::parseTokens (childArgs, emptyEnv()); + expect (! r.handled); + expectEquals (r.settings.strictnessLevel, 7); + } + + // Combining it with any other option is rejected. + { + childArgs.addArray ({ "--strictness-level", "9" }); + const auto r = settings_parser::parseTokens (childArgs, emptyEnv()); + expect (r.handled); + expectEquals (r.exitCode, 1); + } + } + + beginTest ("Subcommand: validate with a positional plugin path"); + { + const auto currentDir = juce::File::getCurrentWorkingDirectory(); + + // Plugin path as a positional after the verb. + expectEquals (juce::String (parse ("validate MyPlugin.vst3").validatePath), + currentDir.getChildFile ("MyPlugin.vst3").getFullPathName()); + + // Options before the positional plugin path. + const auto s = parse ("validate --strictness-level 8 MyPlugin.vst3"); + expectEquals (s.strictnessLevel, 8); + expectEquals (juce::String (s.validatePath), currentDir.getChildFile ("MyPlugin.vst3").getFullPathName()); + + // A bare AU component id positional is left untouched. + expectEquals (juce::String (parse ("validate MyPluginID").validatePath), juce::String ("MyPluginID")); + } + + beginTest ("Subcommand: validate with --config before the positional plugin"); + { + juce::TemporaryFile configFile (".json"); + configFile.getFile().replaceWithText (R"({ "strictnessLevel": 2 })"); + + const auto currentDir = juce::File::getCurrentWorkingDirectory(); + const auto cmd = "validate --config " + configFile.getFile().getFullPathName().quoted() + " MyPlugin.vst3"; + + const auto s = parse (cmd); + expectEquals (s.strictnessLevel, 2); // --config didn't swallow the plugin path + expectEquals (juce::String (s.validatePath), currentDir.getChildFile ("MyPlugin.vst3").getFullPathName()); + } + + beginTest ("Subcommand dispatch routing (new verbs do not warn)"); + { + using settings_parser::Command; + + { + const auto d = settings_parser::dispatch (settings_parser::tokenise ("run-tests")); + expect (d.command == Command::runTests); + expect (! d.deprecatedAlias); + } + { + const auto d = settings_parser::dispatch (settings_parser::tokenise ("strictness-help 7")); + expect (d.command == Command::strictnessHelp); + expectEquals (d.strictnessLevel, 7); + expect (! d.deprecatedAlias); + } + { + const auto d = settings_parser::dispatch (settings_parser::tokenise ("strictness-help")); + expect (d.command == Command::strictnessHelp); + expectEquals (d.strictnessLevel, 5); // default level + } + { + const auto d = settings_parser::dispatch (settings_parser::tokenise ("validate MyPlugin.vst3")); + expect (d.command == Command::validate); + expect (! d.deprecatedAlias); + } + } + + beginTest ("Deprecated flat flags still route, flagged as deprecated"); + { + using settings_parser::Command; + + { + const auto d = settings_parser::dispatch (settings_parser::tokenise ("--run-tests")); + expect (d.command == Command::runTests); + expect (d.deprecatedAlias); + } + { + const auto d = settings_parser::dispatch (settings_parser::tokenise ("--strictness-help 9")); + expect (d.command == Command::strictnessHelp); + expectEquals (d.strictnessLevel, 9); + expect (d.deprecatedAlias); + } + { + const auto d = settings_parser::dispatch (settings_parser::tokenise ("--validate x")); + expect (d.command == Command::validate); + expect (d.deprecatedAlias); + } + } + + beginTest ("Bare-path shorthand and child handoff do not warn"); + { + juce::TemporaryFile temp ("path_to_file.vst3"); + expect (temp.getFile().create()); + + // Bare plugin path -> validate, no deprecation warning. + const auto bare = settings_parser::dispatch (settings_parser::tokenise (temp.getFile().getFullPathName())); + expect (bare.command == settings_parser::Command::validate); + expect (! bare.deprecatedAlias); + + // The internal child handoff uses the explicit verb -> no warning. + PluginTests::Options opts; + juce::StringArray childArgs (createCommandLine ("/some/MyPlugin.vst3", opts)); + childArgs.remove (0); // drop the executable path + const auto child = settings_parser::dispatch (childArgs); + expect (child.command == settings_parser::Command::validate); + expect (! child.deprecatedAlias); + } + + beginTest ("Should perform command line recognises subcommands"); + { + expect (shouldPerformCommandLine ("run-tests")); + expect (shouldPerformCommandLine ("strictness-help")); + expect (shouldPerformCommandLine ("validate MyPlugin.vst3")); + expect (shouldPerformCommandLine ("validate MyPluginID")); + expect (shouldPerformCommandLine ("test config.json")); + } + + beginTest ("Subcommand: test captures the positional config path"); + { + using settings_parser::Command; + + { + const auto d = settings_parser::dispatch (settings_parser::tokenise ("test config.json")); + expect (d.command == Command::test); + expect (! d.deprecatedAlias); + expectEquals (d.testConfigPath, juce::String ("config.json")); + } + { + // The config path is the first non-option token after the verb. + const auto d = settings_parser::dispatch (settings_parser::tokenise ("test ./refs/sine.json")); + expect (d.command == Command::test); + expectEquals (d.testConfigPath, juce::String ("./refs/sine.json")); + } + { + // Missing positional -> empty path (the runner reports the usage error). + const auto d = settings_parser::dispatch (settings_parser::tokenise ("test")); + expect (d.command == Command::test); + expect (d.testConfigPath.isEmpty()); + } + } + + beginTest ("Subcommand: test options (--record-missing, --timeout-ms)"); + { + using settings_parser::Command; + + { + const auto d = settings_parser::dispatch (settings_parser::tokenise ("test --record-missing --timeout-ms 500 config.json")); + expect (d.command == Command::test); + expect (d.recordMissing); + expectEquals (d.timeoutMs, (juce::int64) 500); + expectEquals (d.testConfigPath, juce::String ("config.json")); + } + { + // Options after the positional; the timeout value isn't taken as the config. + const auto d = settings_parser::dispatch (settings_parser::tokenise ("test config.json --timeout-ms -1")); + expectEquals (d.testConfigPath, juce::String ("config.json")); + expectEquals (d.timeoutMs, (juce::int64) -1); + } + { + const auto d = settings_parser::dispatch (settings_parser::tokenise ("test --timeout-ms 500 config.json")); + expectEquals (d.testConfigPath, juce::String ("config.json")); + expect (d.error.isEmpty()); + } + { + // Unknown options are errors, not silently skipped (their value + // would otherwise be taken as the config path). + const auto d = settings_parser::dispatch (settings_parser::tokenise ("test --output-dir out config.json")); + expect (d.error.contains ("--output-dir")); + } + { + const auto d = settings_parser::dispatch (settings_parser::tokenise ("test config.json --timeout-ms abc")); + expect (d.error.contains ("--timeout-ms")); + } + { + const auto d = settings_parser::dispatch (settings_parser::tokenise ("test config.json --timeout-ms")); + expect (d.error.contains ("--timeout-ms")); + } + { + const auto d = settings_parser::dispatch (settings_parser::tokenise ("test a.json b.json")); + expect (d.error.contains ("b.json")); + } + } + + beginTest ("Acceptance TestConfig JSON parsing (snake_case keys, defaults)"); + { + const auto json = R"({ + "name": "myReverb-default", + "plugin": "/path/to/Plugin.vst3", + "input": { "audio": "in.wav" }, + "state": { "parameters": { "Mix": 0.5, "3": 1.0 } }, + "sample_rate": 48000, + "block_size": 256, + "render_duration": 2.0 + })"; + + const auto config = nlohmann::ordered_json::parse (json).get(); + + expectEquals (config.getName(), juce::String ("myReverb-default")); + expectEquals (juce::String (config.plugin), juce::String ("/path/to/Plugin.vst3")); + expectEquals (juce::String (config.inputAudio), juce::String ("in.wav")); + expect (config.inputMidi.empty()); + expectEquals (config.sampleRate, 48000.0); + expectEquals (config.blockSize, 256); + expect (config.renderDuration.has_value()); + expectEquals (*config.renderDuration, 2.0); + // state.parameters keeps file order ("Mix" sorts after "3"). + expectEquals ((int) config.stateParameters.size(), 2); + expectEquals (juce::String (config.stateParameters[0].first), juce::String ("Mix")); + expectEquals (config.stateParameters[0].second, 0.5); + expectEquals (juce::String (config.stateParameters[1].first), juce::String ("3")); + expectEquals (config.stateParameters[1].second, 1.0); + + // Omitted comparison falls back to one 16-bit LSB. + const auto comparison = config.getComparison(); + expect (comparison.contains ("sample")); + expectEquals (comparison["sample"].get(), 1.0 / 32768.0); + } + + beginTest ("Acceptance TestConfig omitted render_duration and explicit comparison"); + { + const auto json = R"({ + "name": "t", + "plugin": "Plugin.vst3", + "comparison": { "sample": 0.0 } + })"; + + const auto config = nlohmann::ordered_json::parse (json).get(); + expect (! config.renderDuration.has_value()); + expectEquals (config.getComparison()["sample"].get(), 0.0); + } + + beginTest ("Acceptance TestConfig playhead (object time signature)"); + { + // Absent playhead -> unset (no transport supplied to the plugin). + { + const auto config = nlohmann::ordered_json::parse (R"({ "name": "t", "plugin": "P.vst3" })").get(); + expect (! config.playhead.has_value()); + } + + // Present playhead -> parsed, with time_signature as an object. + { + const auto json = R"({ + "name": "t", + "plugin": "P.vst3", + "playhead": { + "bpm": 90, + "time_signature": { "numerator": 6, "denominator": 8 }, + "start_ppq": 4.0 + } + })"; + + const auto config = nlohmann::ordered_json::parse (json).get(); + expect (config.playhead.has_value()); + expectEquals (config.playhead->bpm, 90.0); + expectEquals (config.playhead->timeSigNumerator, 6); + expectEquals (config.playhead->timeSigDenominator, 8); + expectEquals (config.playhead->startPpq, 4.0); + } + + // Time signature defaults to 4/4 when omitted. + { + const auto config = nlohmann::ordered_json::parse (R"({ "name": "t", "plugin": "P.vst3", "playhead": { "bpm": 100 } })") + .get(); + expect (config.playhead.has_value()); + expectEquals (config.playhead->timeSigNumerator, 4); + expectEquals (config.playhead->timeSigDenominator, 4); + } + + // An invalid (zero) denominator is rejected. + { + bool threw = false; + try + { + nlohmann::ordered_json::parse (R"({ "name": "t", "plugin": "P.vst3", "playhead": { "bpm": 120, "time_signature": { "numerator": 4, "denominator": 0 } } })") + .get(); + } + catch (const std::exception&) { threw = true; } + expect (threw, "expected a zero denominator to be rejected"); + } + } + + beginTest ("Acceptance TestConfig validation"); + { + const auto rejects = [] (const char* json) + { + try + { + nlohmann::ordered_json::parse (json).get(); + } + catch (const std::exception&) + { + return true; + } + + return false; + }; + + expect (! rejects (R"({ "name": "t", "plugin": "P.vst3" })")); + expect (! rejects (R"({ "reference": "r.wav", "plugin": "P.vst3" })"), "reference alone names the test"); + + expect (rejects (R"({ "name": "t" })"), "plugin is required"); + expect (rejects (R"({ "plugin": "P.vst3" })"), "name or reference is required"); + expect (rejects (R"({ "name": "t", "plugin": "P.vst3", "comparison": {} })"), "empty comparison"); + expect (rejects (R"({ "name": "t", "plugin": "P.vst3", "comparison": 1 })"), "non-object comparison"); + expect (rejects (R"({ "name": "t", "plugin": "P.vst3", "sample_rate": 0 })"), "zero sample_rate"); + expect (rejects (R"({ "name": "t", "plugin": "P.vst3", "sample_rate": -48000 })"), "negative sample_rate"); + expect (rejects (R"({ "name": "t", "plugin": "P.vst3", "block_size": 0 })"), "zero block_size"); + expect (rejects (R"({ "name": "t", "plugin": "P.vst3", "block_size": 65537 })"), "huge block_size"); + expect (rejects (R"({ "name": "t", "plugin": "P.vst3", "render_duration": 0 })"), "zero render_duration"); + expect (rejects (R"({ "name": "t", "plugin": "P.vst3", "render_duration": "1" })"), "string render_duration"); + expect (rejects (R"({ "name": "t", "plugin": "P.vst3", "render_duration": 1e6, "sample_rate": 48000 })"), + "render_duration * sample_rate overflowing int"); + } + + beginTest ("Acceptance sample comparator fails on NaN"); + { + auto comparator = acceptance::createComparator ("sample"); + expect (comparator != nullptr); + + juce::AudioBuffer reference (1, 4), output (1, 4); + reference.clear(); + output.clear(); + + expect (comparator->compare (reference, output, nlohmann::ordered_json (0.0)).passed); + + output.setSample (0, 2, std::numeric_limits::quiet_NaN()); + const auto r = comparator->compare (reference, output, nlohmann::ordered_json (1.0)); + expect (! r.passed, "NaN must not pass, whatever the tolerance"); + expectEquals (r.details["first_fail_channel"].get(), 0); + expectEquals (r.details["first_fail_sample"].get(), 2); + } + } +}; + +static CommandLineTests commandLineTests; diff --git a/source/CrashHandler.cpp b/source/CrashHandler.cpp new file mode 100644 index 00000000..d56c8937 --- /dev/null +++ b/source/CrashHandler.cpp @@ -0,0 +1,81 @@ +/*============================================================================== + + Copyright 2018 by Tracktion Corporation. + For more information visit www.tracktion.com + + You may also use this code under the terms of the GPL v3 (see + www.gnu.org/licenses). + + pluginval IS PROVIDED "AS IS" WITHOUT ANY WARRANTY, AND ALL WARRANTIES, WHETHER + EXPRESSED OR IMPLIED, INCLUDING MERCHANTABILITY AND FITNESS FOR PURPOSE, ARE + DISCLAIMED. + + ==============================================================================*/ + +#include "juce_core/juce_core.h" +#include "CrashHandler.h" + +#if ! JUCE_WINDOWS + #include "native/pluginval_native_posix.h" + static const auto crashLogPath = "/tmp/pluginval_crash.txt"; +#endif + +namespace +{ + juce::String getCrashLogContents() + { + return "\n" + juce::SystemStats::getStackBacktrace(); + } + + static juce::File getCrashTraceFile() + { + #if JUCE_WINDOWS + return juce::File::getSpecialLocation (juce::File::tempDirectory).getChildFile ("pluginval_crash.txt"); + #else + return juce::File (crashLogPath); + #endif + } + + #if JUCE_WINDOWS + static void handleCrash (void*) + { + const auto log = getCrashLogContents(); + std::cout << "\n*** FAILED: VALIDATION CRASHED\n" << log << std::endl; + getCrashTraceFile().replaceWithText (log); + } + #else + static void handleCrash (void*) + { + const char* header = "\n*** FAILED: VALIDATION CRASHED\n"; + [[ maybe_unused]] auto r = write (STDERR_FILENO, header, strlen (header)); + + writeStackTrace (crashLogPath); + + // Terminate normally to work around a bug in juce::ChildProcess::ActiveProcess::getExitStatus() + // which returns 0 (a "pass" in the host process) if the child process terminates abnormally. + // - https://github.com/Tracktion/pluginval/issues/125 + // - https://forum.juce.com/t/killed-childprocess-activeprocess-exit-code/61645/3 + // Use _Exit() instead of exit() so that static destructors don't run (they may not be async-signal-safe). + // FIXME: exiting here prevents Apple's Crash Reporter from creating reports. + std::_Exit (SIGKILL); + } + #endif +} + +//============================================================================== +void initialiseCrashHandler() +{ + // Delete the crash file, this will be created if possible + getCrashTraceFile().deleteFile(); + juce::SystemStats::setApplicationCrashHandler (handleCrash); +} + +juce::String getCrashLog() +{ + const auto f = getCrashTraceFile(); + + if (f.existsAsFile()) + return f.loadFileAsString(); + + return getCrashLogContents(); +} diff --git a/Source/CrashHandler.h b/source/CrashHandler.h similarity index 100% rename from Source/CrashHandler.h rename to source/CrashHandler.h diff --git a/Source/Main.cpp b/source/Main.cpp similarity index 100% rename from Source/Main.cpp rename to source/Main.cpp diff --git a/Source/MainComponent.cpp b/source/MainComponent.cpp similarity index 100% rename from Source/MainComponent.cpp rename to source/MainComponent.cpp diff --git a/Source/MainComponent.h b/source/MainComponent.h similarity index 100% rename from Source/MainComponent.h rename to source/MainComponent.h diff --git a/Source/PluginTests.cpp b/source/PluginTests.cpp similarity index 100% rename from Source/PluginTests.cpp rename to source/PluginTests.cpp diff --git a/Source/PluginTests.h b/source/PluginTests.h similarity index 100% rename from Source/PluginTests.h rename to source/PluginTests.h diff --git a/Source/PluginvalLookAndFeel.h b/source/PluginvalLookAndFeel.h similarity index 100% rename from Source/PluginvalLookAndFeel.h rename to source/PluginvalLookAndFeel.h diff --git a/source/PluginvalSettings.h b/source/PluginvalSettings.h new file mode 100644 index 00000000..e8a82558 --- /dev/null +++ b/source/PluginvalSettings.h @@ -0,0 +1,142 @@ +/*============================================================================== + + Copyright 2018 by Tracktion Corporation. + For more information visit www.tracktion.com + + You may also use this code under the terms of the GPL v3 (see + www.gnu.org/licenses). + + pluginval IS PROVIDED "AS IS" WITHOUT ANY WARRANTY, AND ALL WARRANTIES, WHETHER + EXPRESSED OR IMPLIED, INCLUDING MERCHANTABILITY AND FITNESS FOR PURPOSE, ARE + DISCLAIMED. + + ==============================================================================*/ + +#pragma once + +#include "PluginTests.h" +#include + +#include +#include +#include + +//============================================================================== +/** Maps the RealtimeCheck enum to/from its JSON string representation. + An unrecognised/empty string deserialises to the first entry (disabled), + preserving the historical "empty -> disabled" behaviour. +*/ +NLOHMANN_JSON_SERIALIZE_ENUM (RealtimeCheck, +{ + { RealtimeCheck::disabled, "disabled" }, + { RealtimeCheck::enabled, "enabled" }, + { RealtimeCheck::relaxed, "relaxed" }, +}) + +//============================================================================== +/** + The single, parser-agnostic settings struct. + + Every field uses std types so it can be (de)serialised by nlohmann/json and + is decoupled from JUCE. JSON keys mirror the field names below. Conversion to + the JUCE-flavoured PluginTests::Options happens only at the boundary via + toPluginTestOptions(). +*/ +struct PluginvalSettings +{ + int strictnessLevel = 5; + std::int64_t randomSeed = 0; + std::int64_t timeoutMs = 30000; + bool verbose = false; + int numRepeats = 1; + bool randomiseTestOrder = false; + bool skipGuiTests = false; + std::string dataFile; + std::string outputDir; + std::string outputFilename; + std::vector disabledTests; + std::vector sampleRates; + std::vector blockSizes; + RealtimeCheck realtimeCheck = RealtimeCheck::disabled; + std::string validatePath; + + bool operator== (const PluginvalSettings&) const = default; + + //============================================================================== + /** The default sample rates used when none are specified. */ + static std::vector defaultSampleRates() { return { 44100.0, 48000.0, 96000.0 }; } + /** The default block sizes used when none are specified. */ + static std::vector defaultBlockSizes() { return { 64, 128, 256, 512, 1024 }; } + + //============================================================================== + /** Converts to the JUCE-flavoured options consumed by PluginTests. */ + PluginTests::Options toPluginTestOptions() const + { + PluginTests::Options o; + o.strictnessLevel = juce::jlimit (1, 10, strictnessLevel); + o.randomSeed = randomSeed; + o.timeoutMs = timeoutMs; + o.verbose = verbose; + o.numRepeats = juce::jmax (1, numRepeats); + o.randomiseTestOrder = randomiseTestOrder; + o.withGUI = ! skipGuiTests; + o.dataFile = juce::String (dataFile); + o.outputDir = juce::String (outputDir); + o.outputFilename = juce::String (outputFilename); + o.disabledTests = toStringArray (disabledTests); + o.sampleRates = sampleRates.empty() ? defaultSampleRates() : sampleRates; + o.blockSizes = blockSizes.empty() ? defaultBlockSizes() : blockSizes; + o.realtimeCheck = realtimeCheck; + return o; + } + + /** Builds settings from a resolved PluginTests::Options + plugin path. + Used to serialise options for the child validation process. + */ + static PluginvalSettings fromPluginTestOptions (const PluginTests::Options& o, const juce::String& fileOrID) + { + PluginvalSettings s; + s.strictnessLevel = o.strictnessLevel; + s.randomSeed = o.randomSeed; + s.timeoutMs = o.timeoutMs; + s.verbose = o.verbose; + s.numRepeats = o.numRepeats; + s.randomiseTestOrder = o.randomiseTestOrder; + s.skipGuiTests = ! o.withGUI; + s.dataFile = o.dataFile.getFullPathName().toStdString(); + s.outputDir = o.outputDir.getFullPathName().toStdString(); + s.outputFilename = o.outputFilename.toStdString(); + s.disabledTests = fromStringArray (o.disabledTests); + s.sampleRates = o.sampleRates; + s.blockSizes = o.blockSizes; + s.realtimeCheck = o.realtimeCheck; + s.validatePath = fileOrID.toStdString(); + return s; + } + +private: + static juce::StringArray toStringArray (const std::vector& v) + { + juce::StringArray a; + for (const auto& s : v) + a.add (juce::String (s)); + return a; + } + + static std::vector fromStringArray (const juce::StringArray& a) + { + std::vector v; + for (const auto& s : a) + v.push_back (s.toStdString()); + return v; + } +}; + +//============================================================================== +// to_json/from_json with per-field defaults: keys missing from the JSON fall +// back to a default-constructed PluginvalSettings rather than value-zero. This +// makes "defaults" a free layer and lets sparse JSON layers merge cleanly. +NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE_WITH_DEFAULT (PluginvalSettings, + strictnessLevel, randomSeed, timeoutMs, verbose, numRepeats, + randomiseTestOrder, skipGuiTests, dataFile, outputDir, outputFilename, + disabledTests, sampleRates, blockSizes, realtimeCheck, validatePath) diff --git a/Source/RTCheck.h b/source/RTCheck.h similarity index 100% rename from Source/RTCheck.h rename to source/RTCheck.h diff --git a/source/SettingsParser.cpp b/source/SettingsParser.cpp new file mode 100644 index 00000000..45b61356 --- /dev/null +++ b/source/SettingsParser.cpp @@ -0,0 +1,569 @@ +/*============================================================================== + + Copyright 2018 by Tracktion Corporation. + For more information visit www.tracktion.com + + You may also use this code under the terms of the GPL v3 (see + www.gnu.org/licenses). + + pluginval IS PROVIDED "AS IS" WITHOUT ANY WARRANTY, AND ALL WARRANTIES, WHETHER + EXPRESSED OR IMPLIED, INCLUDING MERCHANTABILITY AND FITNESS FOR PURPOSE, ARE + DISCLAIMED. + + ==============================================================================*/ + +// CLI11 must be included before the JUCE headers: on Linux JUCE pulls in the X11 +// headers (JUCE_GUI_BASICS_INCLUDE_XHEADERS), which #define Success/None/Bool/etc. +// and would clash with CLI11's CLI::ExitCodes::Success enumerator. +#include + +#include "SettingsParser.h" +#include "SettingsSerializer.h" + +#include +#include +#include +#include +#include +#include + +namespace settings_parser +{ + juce::String systemEnv (const juce::String& name) + { + return juce::SystemStats::getEnvironmentVariable (name, {}); + } + + juce::String getVersionString() + { + return juce::String ("pluginval") + " - " + VERSION; + } + + //============================================================================== + namespace + { + bool isPluginArgument (const juce::String& arg) + { + juce::AudioPluginFormatManager formatManager; + #if JUCE_VERSION >= 0x08000B + juce::addDefaultFormatsToManager (formatManager); + #else + formatManager.addDefaultFormats(); + #endif + + for (auto format : formatManager.getFormats()) + if (format->fileMightContainThisPluginType (arg)) + return true; + + // The above checks the file exists, which isn't what we want for CLI parsing + if (auto f = juce::File::createFileWithoutCheckingPath (arg); + f.hasFileExtension (".vst3") + #if JUCE_PLUGINHOST_VST + || f.hasFileExtension (".dll") + #endif + ) + return true; + + return false; + } + + juce::String valueForOption (const juce::StringArray& tokens, juce::StringRef option) + { + const juce::String prefix = juce::String (option) + "="; + + for (int i = 0; i < tokens.size(); ++i) + { + if (tokens[i] == option) + return i + 1 < tokens.size() ? tokens[i + 1] : juce::String(); + + if (tokens[i].startsWith (prefix)) + return tokens[i].substring (prefix.length()); + } + + return {}; + } + + /** Returns every value supplied for an option, in command-line order. */ + juce::StringArray allValuesForOption (const juce::StringArray& tokens, juce::StringRef option) + { + juce::StringArray out; + const juce::String prefix = juce::String (option) + "="; + + for (int i = 0; i < tokens.size(); ++i) + { + if (tokens[i] == option) + { + if (i + 1 < tokens.size()) + out.add (tokens[i + 1]); + } + else if (tokens[i].startsWith (prefix)) + { + out.add (tokens[i].substring (prefix.length())); + } + } + + return out; + } + + bool hasOption (const juce::StringArray& tokens, juce::StringRef option) + { + const juce::String prefix = juce::String (option) + "="; + + for (const auto& t : tokens) + if (t == option || t.startsWith (prefix)) + return true; + + return false; + } + + juce::String decodeBase64 (const juce::String& b64) + { + juce::MemoryOutputStream mos; + juce::Base64::convertFromBase64 (mos, b64); + return mos.toString(); + } + + juce::String getFooterText() + { + return juce::SystemStats::getJUCEVersion() + "\n\n" + juce::String ( +R"(Commands: + validate [options] Validate the plugin at the given path or AU id (the default). + run-tests Run the internal unit tests. + strictness-help [level] List all tests that run at the given strictness level. + test [options] Run a deterministic acceptance (golden-file) test from a config. + --record-missing Record a missing reference and pass + (default: a missing reference fails). + --timeout-ms Fail if the run takes longer than this + (default 30000, -1 to never time out). + +The flat flags --validate , --run-tests and --strictness-help [level] are +deprecated aliases for the commands above and will be removed in a future version. + +Exit code: + 0 if all tests complete successfully + 1 if there are any errors + +You can also specify any option as an environment variable by removing the prefix +dashes, converting internal dashes to underscores and capitalising, e.g. + "--skip-gui-tests" -> "SKIP_GUI_TESTS=1" + "--timeout-ms 30000" -> "TIMEOUT_MS=30000" +Precedence (lowest to highest): defaults, environment variables, --config, command-line options. +--config is repeatable; later files win per key.)"); + } + + std::string toEnvName (const std::string& longName) + { + return juce::String (longName).toUpperCase().replace ("-", "_").toStdString(); + } + + /** Loads one --config JSON file. */ + nlohmann::json loadConfigFile (const juce::String& path) + { + const juce::File file (path); + + if (! file.existsAsFile()) + throw std::runtime_error (("--config file not found: " + path).toStdString()); + + return nlohmann::json::parse (file.loadFileAsString().toStdString()); + } + + /** Registers every option, bound to s. No ->envname(): the environment is a + separate, lower-precedence layer (see parseTokens). */ + void configureApp (CLI::App& app, PluginvalSettings& s) + { + app.set_version_flag ("--version", getVersionString().toStdString()); + app.footer (getFooterText().toStdString()); + + // Accepted so the CLI parse doesn't error on --config; the values are + // handled manually (repeatable, inline-or-file) in parseTokens. + app.add_option_function> ("--config", + [] (const std::vector&) {}, + "Path to a JSON settings file. Repeatable; later files win per key.")->take_all(); + + app.add_option ("--validate", s.validatePath, "Validates the plugin at the given path (or AU id)."); + // The new canonical form: "pluginval validate ". Bound to the + // same member as --validate; whichever is supplied sets the path. + app.add_option ("plugin", s.validatePath, "Plugin path or AU id to validate."); + app.add_option ("--strictness-level", s.strictnessLevel, "Strictness level 1-10 (default 5)."); + app.add_option ("--timeout-ms", s.timeoutMs, "Test timeout in ms (default 30000, -1 to never timeout)."); + app.add_option ("--repeat", s.numRepeats, "Number of times to repeat the tests."); + app.add_flag ("--randomise", s.randomiseTestOrder, "Run the tests in a random order per repeat."); + app.add_flag ("--verbose", s.verbose, "Output additional logging information."); + app.add_flag ("--skip-gui-tests", s.skipGuiTests, "Avoid tests that create GUI windows (for headless CI)."); + app.add_option ("--sample-rates", s.sampleRates, "Comma-separated sample rates (default 44100,48000,96000).")->delimiter (','); + app.add_option ("--block-sizes", s.blockSizes, "Comma-separated block sizes (default 64,128,256,512,1024).")->delimiter (','); + app.add_option ("--data-file", s.dataFile, "Path to a data file tests can use to configure themselves."); + app.add_option ("--output-dir", s.outputDir, "Directory in which to write the log files."); + app.add_option ("--output-filename", s.outputFilename, "Filename to write logs into."); + + app.add_option_function ("--disabled-tests", + [&s] (const std::string& v) { s.disabledTests = settings_serializer::disabledTestsToList (juce::String (v)); }, + "Comma-separated test names, or a path to a file listing them."); + + app.add_option_function ("--random-seed", + [&s] (const std::string& v) + { + try { s.randomSeed = settings_serializer::parseRandomSeed (juce::String (v)); } + catch (const std::exception& e) { throw CLI::ValidationError ("--random-seed", e.what()); } + }, + "Random seed (hex 0x.. or int) for replicable test runs."); + + app.add_option ("--rtcheck", s.realtimeCheck, "Real-time safety checks: disabled, enabled or relaxed.") + ->transform (CLI::CheckedTransformer (std::map { + { "disabled", RealtimeCheck::disabled }, + { "enabled", RealtimeCheck::enabled }, + { "relaxed", RealtimeCheck::relaxed } }, CLI::ignore_case)); + } + + /** Builds a synthetic argv from the environment by deriving an env-var name + from each registered option (e.g. --strictness-level -> STRICTNESS_LEVEL). + CLI11 then parses and coerces it like any other argument. */ + std::vector buildEnvArgv (const CLI::App& app, const EnvProvider& env) + { + std::vector argv { "pluginval" }; + + for (const auto* opt : app.get_options()) + { + const auto& lnames = opt->get_lnames(); + + if (lnames.empty()) + continue; + + const auto& lname = lnames.front(); + + // Skip the meta options that shouldn't be environment-driven. + if (lname == "help" || lname == "version" || lname == "config" || lname == "validate") + continue; + + if (const auto value = env (juce::String (toEnvName (lname))); value.isNotEmpty()) + argv.push_back ("--" + lname + "=" + value.toStdString()); // works for flags too (--flag=1/0) + } + + return argv; + } + + /** Runs one parse pass. Returns an exit code if the parse was "handled" + (help/version/error), or nullopt on success. */ + std::optional runParse (CLI::App& app, const std::vector& argv) + { + std::vector cargv; + cargv.reserve (argv.size()); + + for (const auto& a : argv) + cargv.push_back (a.c_str()); + + try + { + app.parse ((int) cargv.size(), cargv.data()); + } + catch (const CLI::ParseError& e) + { + return app.exit (e); + } + + return std::nullopt; + } + } + + //============================================================================== + juce::StringArray tokenise (const juce::String& commandLineIn) + { + if (commandLineIn.contains ("strictnessLevel")) + { + std::cout << "!!! WARNING:\n\t\"strictnessLevel\" is deprecated and will be removed in a future version.\n" + << "\tPlease use --strictness-level instead\n\n"; + } + + const auto commandLine = commandLineIn.replace ("strictnessLevel", "strictness-level") + .replace ("-NSDocumentRevisionsDebugMode YES", "") + .trim(); + + juce::StringArray args; + args.addTokens (commandLine, true); + args.trim(); + + for (auto& s : args) + s = s.unquoted(); + + return args; + } + + void insertImplicitValidate (juce::StringArray& args) + { + // If only a plugin path is supplied as the last arg, add an implicit + // --validate option for it so the rest of the CLI works. + if (args.size() > 0) + { + const bool hasCommand = hasOption (args, "--validate") + || args.contains ("--help") || args.contains ("-h") + || args.contains ("--version") + || args.contains ("--run-tests") + || hasOption (args, "--config-base64"); + + if (! hasCommand && isPluginArgument (args[args.size() - 1])) + args.insert (args.size() - 1, "--validate"); + } + } + + juce::StringArray preprocess (const juce::String& commandLine) + { + auto args = tokenise (commandLine); + insertImplicitValidate (args); + return args; + } + + bool isCommandLine (const juce::StringArray& tokens) + { + if (! tokens.isEmpty()) + { + const auto& verb = tokens.getReference (0); + + if (verb == "validate" || verb == "run-tests" || verb == "strictness-help" || verb == "test") + return true; + } + + return tokens.contains ("--help") || tokens.contains ("-h") + || tokens.contains ("--version") + || hasOption (tokens, "--validate") + || tokens.contains ("--run-tests") + || tokens.contains ("--strictness-help") + || hasOption (tokens, "--config-base64"); + } + + //============================================================================== + DispatchResult dispatch (const juce::StringArray& tokensIn) + { + DispatchResult result; + + // The optional [level] argument that follows the strictness-help command/flag. + const auto levelAfter = [] (const juce::StringArray& tokens, int idx) + { + if (idx >= 0 && idx + 1 < tokens.size()) + if (const auto& next = tokens.getReference (idx + 1); ! next.startsWith ("-")) + return juce::jlimit (1, 10, next.getIntValue()); + + return 5; + }; + + // 1. Explicit subcommand verbs (the new, non-deprecated syntax). + if (! tokensIn.isEmpty()) + { + const auto& verb = tokensIn.getReference (0); + + if (verb == "validate") + { + result.command = Command::validate; + result.validateTokens = tokensIn; + result.validateTokens.remove (0); // the plugin path is captured by the positional CLI option + return result; + } + + if (verb == "run-tests") + { + result.command = Command::runTests; + return result; + } + + if (verb == "strictness-help") + { + result.command = Command::strictnessHelp; + result.strictnessLevel = levelAfter (tokensIn, 0); + return result; + } + + if (verb == "test") + { + result.command = Command::test; + + // Environment variables first, so the command line wins (matching validate). + if (const auto env = juce::SystemStats::getEnvironmentVariable ("RECORD_MISSING", {}); env.isNotEmpty()) + result.recordMissing = env.getIntValue() != 0 || env.equalsIgnoreCase ("true"); + + if (const auto env = juce::SystemStats::getEnvironmentVariable ("TIMEOUT_MS", {}); env.isNotEmpty()) + result.timeoutMs = env.getLargeIntValue(); + + // The options, plus the positional acceptance-test config (the first + // non-option token after the verb). + for (int i = 1; i < tokensIn.size(); ++i) + { + const auto& token = tokensIn.getReference (i); + + if (token == "--record-missing") + { + result.recordMissing = true; + } + else if (token == "--timeout-ms") + { + const auto value = i + 1 < tokensIn.size() ? tokensIn.getReference (++i) : juce::String(); + const auto digits = value.startsWith ("-") ? value.substring (1) : value; + + if (digits.isEmpty() || ! digits.containsOnly ("0123456789")) + { + result.error = "--timeout-ms needs a whole number of milliseconds (got \"" + value + "\")"; + break; + } + + result.timeoutMs = value.getLargeIntValue(); + } + else if (token.startsWith ("-")) + { + result.error = "unknown option for test: " + token; + break; + } + else if (result.testConfigPath.isEmpty()) + { + result.testConfigPath = token; + } + else + { + result.error = "unexpected argument for test: " + token + " (only one config can be given)"; + break; + } + } + + return result; + } + } + + // 2. Deprecated flat command flags. + if (tokensIn.contains ("--run-tests")) + { + result.command = Command::runTests; + result.deprecatedAlias = true; + return result; + } + + if (tokensIn.contains ("--strictness-help")) + { + result.command = Command::strictnessHelp; + result.deprecatedAlias = true; + result.strictnessLevel = levelAfter (tokensIn, tokensIn.indexOf ("--strictness-help")); + return result; + } + + // 3. Default: validate. An explicit --validate flag (but not the bare-path + // shorthand, nor the internal --config-base64 handoff) is a deprecated alias. + result.command = Command::validate; + result.deprecatedAlias = hasOption (tokensIn, "--validate") && ! hasOption (tokensIn, "--config-base64"); + result.validateTokens = tokensIn; + insertImplicitValidate (result.validateTokens); + return result; + } + + juce::String resolvePluginPath (const juce::String& raw) + { + // Resolve relative/home paths against the working directory. Absolute + // paths and bare component IDs (no '.' or '~') are left untouched. + if (raw.contains ("~") || raw.contains (".")) + return juce::File::getCurrentWorkingDirectory().getChildFile (raw).getFullPathName(); + + return raw; + } + + //============================================================================== + ParseResult parseTokens (const juce::StringArray& tokens, const EnvProvider& env) + { + ParseResult result; + auto& s = result.settings; + + // --config-base64 carries a fully-resolved settings set from the parent + // process and is authoritative. It is for internal use only and must not be + // combined with other options (which would otherwise be silently ignored); + // the only companion allowed is --validate. + if (const auto b64 = valueForOption (tokens, "--config-base64"); b64.isNotEmpty()) + { + for (const auto& token : tokens) + { + if (! token.startsWith ("-") || token == "-") + continue; // a value, not an option + + if (const auto name = token.upToFirstOccurrenceOf ("=", false, false); + name != "--config-base64" && name != "--validate") + { + std::cerr << "*** FAILED: --config-base64 is for internal use and cannot be combined " + << "with other options (got " << name << ")" << std::endl; + result.exitCode = 1; + result.handled = true; + return result; + } + } + + s = settings_serializer::fromJsonString (decodeBase64 (b64).toStdString()); + s.validatePath = resolvePluginPath (juce::String (s.validatePath)).toStdString(); + return result; + } + + // 1. Environment layer (env-var names derived from the registered options). + { + CLI::App envApp; + configureApp (envApp, s); + + if (const auto code = runParse (envApp, buildEnvArgv (envApp, env)); code) + { + result.exitCode = *code; + result.handled = true; + return result; + } + } + + // 2. --config layer: repeatable, inline JSON or file, merged per key in + // command-line order (last wins). Beats the environment, loses to CLI. + if (const auto configs = allValuesForOption (tokens, "--config"); ! configs.isEmpty()) + { + auto merged = settings_serializer::toJson (s); + + for (const auto& source : configs) + merged.merge_patch (loadConfigFile (source)); + + s = merged.get(); + } + + // 3. Command-line layer: the individual options beat everything. + { + CLI::App cliApp { "Validate plugins to test compatibility with hosts and verify plugin API conformance" }; + configureApp (cliApp, s); + + std::vector argv { "pluginval" }; + + // --config is handled manually above; drop it (and its value) here so + // its greedy CLI11 vector parsing can't swallow the positional plugin + // path. It stays registered purely so it appears in --help. + for (int i = 0; i < tokens.size(); ++i) + { + const auto& t = tokens.getReference (i); + + if (t == "--config") { ++i; continue; } // skip the flag and its value + if (t.startsWith ("--config=")) continue; // skip the inline form + + argv.push_back (t.toStdString()); + } + + if (const auto code = runParse (cliApp, argv); code) + { + result.exitCode = *code; + result.handled = true; + return result; + } + } + + s.validatePath = resolvePluginPath (juce::String (s.validatePath)).toStdString(); + return result; + } + + PluginvalSettings parse (const juce::String& commandLine, const EnvProvider& env) + { + return parseTokens (dispatch (tokenise (commandLine)).validateTokens, env).settings; + } + + //============================================================================== + juce::StringArray createChildProcessCommandLine (const juce::String& fileOrID, const PluginTests::Options& options) + { + const auto settings = PluginvalSettings::fromPluginTestOptions (options, fileOrID); + const auto jsonString = settings_serializer::toJson (settings).dump(); + const auto b64 = juce::Base64::toBase64 (juce::String (jsonString)); + + juce::StringArray args (juce::File::getSpecialLocation (juce::File::currentExecutableFile).getFullPathName()); + args.addArray ({ "validate", "--config-base64", b64, fileOrID }); + return args; + } +} diff --git a/source/SettingsParser.h b/source/SettingsParser.h new file mode 100644 index 00000000..5d5ae2a7 --- /dev/null +++ b/source/SettingsParser.h @@ -0,0 +1,119 @@ +/*============================================================================== + + Copyright 2018 by Tracktion Corporation. + For more information visit www.tracktion.com + + You may also use this code under the terms of the GPL v3 (see + www.gnu.org/licenses). + + pluginval IS PROVIDED "AS IS" WITHOUT ANY WARRANTY, AND ALL WARRANTIES, WHETHER + EXPRESSED OR IMPLIED, INCLUDING MERCHANTABILITY AND FITNESS FOR PURPOSE, ARE + DISCLAIMED. + + ==============================================================================*/ + +#pragma once + +#include "PluginvalSettings.h" +#include "PluginTests.h" + +#include + +/** + The command-line -> settings pipeline. + + A single PluginvalSettings is filled by successive layers, lowest to highest + precedence: hardcoded defaults, then environment variables, then --config + (repeatable JSON files, later files win per key), then the individual CLI options. + CLI11 binds each option to a member and also provides the coercion used for + the environment layer (env names are derived from the registered options, so + there is no separate env table). Conversion to the JUCE-flavoured + PluginTests::Options happens at the boundary via toPluginTestOptions(). +*/ +namespace settings_parser +{ + //============================================================================== + /** Returns the value of an environment variable, or "" if unset. */ + using EnvProvider = std::function; + juce::String systemEnv (const juce::String& name); + + //============================================================================== + /** Tokenises a raw command line: rewrites the deprecated "strictnessLevel", + strips the macOS "-NSDocumentRevisionsDebugMode YES" flag, splits and + unquotes. Returns argv tokens (without the program name) exactly as the + user typed them - it does NOT insert an implicit "--validate". + */ + juce::StringArray tokenise (const juce::String& commandLine); + + /** Inserts an implicit "--validate" when the last token is a bare plugin path + (and no command is already present). This is the documented shorthand for + "pluginval ". + */ + void insertImplicitValidate (juce::StringArray& tokens); + + /** Convenience: tokenise + insertImplicitValidate. */ + juce::StringArray preprocess (const juce::String& commandLine); + + /** True if the tokens contain a recognised command (subcommand verb or legacy + flag) that triggers CLI mode. */ + bool isCommandLine (const juce::StringArray& tokens); + + /** Resolves a relative/home plugin path against the working directory, + leaving absolute paths and bare component IDs untouched. + */ + juce::String resolvePluginPath (const juce::String& raw); + + //============================================================================== + /** The subcommand selected on the command line. */ + enum class Command + { + validate, /**< Validate a plugin (the default when no verb is given). */ + runTests, /**< Run the internal unit tests. */ + strictnessHelp, /**< List the tests that run at a given strictness level. */ + test /**< Run a deterministic acceptance (golden-file) test from a config. */ + }; + + /** The outcome of routing tokens to a subcommand. */ + struct DispatchResult + { + Command command = Command::validate; + juce::StringArray validateTokens; /**< Verb-stripped option tokens to feed parseTokens (validate only). */ + bool deprecatedAlias = false; /**< A legacy flat flag (--validate/--run-tests/--strictness-help) was used. */ + int strictnessLevel = 5; /**< The level for the strictnessHelp command. */ + juce::String testConfigPath; /**< The positional config.json for the test command. */ + bool recordMissing = false; /**< test: record a missing reference (and pass) instead of failing. */ + juce::int64 timeoutMs = 30000; /**< test: abort the run after this long (-1 to never time out). */ + juce::String error; /**< test: an invalid option / argument, to report instead of running. */ + }; + + /** Routes tokenised input to a subcommand, peeling the leading verb. The + validate token-layering in parseTokens is left untouched; the plugin path + is captured by a positional CLI option there. Legacy flat flags map onto + the matching command with deprecatedAlias set. + */ + DispatchResult dispatch (const juce::StringArray& tokens); + + //============================================================================== + /** The outcome of parsing the option tokens. */ + struct ParseResult + { + PluginvalSettings settings; + bool handled = false; /**< true if --help/--version/parse-error was handled and the app should exit. */ + int exitCode = 0; /**< the exit code to use when handled is true. */ + }; + + /** Parses preprocessed option tokens into settings (and handles --help/--version). */ + ParseResult parseTokens (const juce::StringArray& tokens, const EnvProvider& env = systemEnv); + + /** Convenience: preprocess + parse from a raw command line, returning settings. */ + PluginvalSettings parse (const juce::String& commandLine, const EnvProvider& env = systemEnv); + + //============================================================================== + /** Serialises options for the child validation process as a base64 JSON + handoff: { exe, validate, --config-base64 , }. + */ + juce::StringArray createChildProcessCommandLine (const juce::String& fileOrID, const PluginTests::Options&); + + /** The "--version" output string. */ + juce::String getVersionString(); +} diff --git a/source/SettingsSerializer.cpp b/source/SettingsSerializer.cpp new file mode 100644 index 00000000..a390b4de --- /dev/null +++ b/source/SettingsSerializer.cpp @@ -0,0 +1,68 @@ +/*============================================================================== + + Copyright 2018 by Tracktion Corporation. + For more information visit www.tracktion.com + + You may also use this code under the terms of the GPL v3 (see + www.gnu.org/licenses). + + pluginval IS PROVIDED "AS IS" WITHOUT ANY WARRANTY, AND ALL WARRANTIES, WHETHER + EXPRESSED OR IMPLIED, INCLUDING MERCHANTABILITY AND FITNESS FOR PURPOSE, ARE + DISCLAIMED. + + ==============================================================================*/ + +#include "SettingsSerializer.h" + +#include + +namespace settings_serializer +{ + nlohmann::json toJson (const PluginvalSettings& s) + { + return nlohmann::json (s); + } + + PluginvalSettings fromJsonString (const std::string& jsonString) + { + return nlohmann::json::parse (jsonString).get(); + } + + PluginvalSettings fromJsonFile (const juce::File& file) + { + return fromJsonString (file.loadFileAsString().toStdString()); + } + + //============================================================================== + std::int64_t parseRandomSeed (const juce::String& raw) + { + if (! raw.containsOnly ("x-0123456789acbdef")) + throw std::invalid_argument ("Invalid random seed argument!"); + + if (raw.startsWith ("0x")) + return raw.getHexValue64(); + + return raw.getLargeIntValue(); + } + + std::vector disabledTestsToList (const juce::String& raw) + { + std::vector out; + + if (juce::File::isAbsolutePath (raw)) + { + juce::StringArray lines; + juce::File (raw).readLines (lines); + + for (const auto& line : lines) + out.push_back (line.toStdString()); + + return out; + } + + for (const auto& token : juce::StringArray::fromTokens (raw, ",", "")) + out.push_back (token.toStdString()); + + return out; + } +} diff --git a/source/SettingsSerializer.h b/source/SettingsSerializer.h new file mode 100644 index 00000000..a5c9336b --- /dev/null +++ b/source/SettingsSerializer.h @@ -0,0 +1,50 @@ +/*============================================================================== + + Copyright 2018 by Tracktion Corporation. + For more information visit www.tracktion.com + + You may also use this code under the terms of the GPL v3 (see + www.gnu.org/licenses). + + pluginval IS PROVIDED "AS IS" WITHOUT ANY WARRANTY, AND ALL WARRANTIES, WHETHER + EXPRESSED OR IMPLIED, INCLUDING MERCHANTABILITY AND FITNESS FOR PURPOSE, ARE + DISCLAIMED. + + ==============================================================================*/ + +#pragma once + +#include "PluginvalSettings.h" +#include + +#include +#include +#include + +/** + JSON (de)serialisation for PluginvalSettings plus the couple of value + conversions the CLI parser still needs (hex seed, disabled-tests file). +*/ +namespace settings_serializer +{ + //============================================================================== + /** Full JSON representation of a settings set (all fields present). */ + nlohmann::json toJson (const PluginvalSettings&); + + /** Parses a complete settings set from a JSON string (missing keys -> defaults). */ + PluginvalSettings fromJsonString (const std::string&); + + /** Parses a complete settings set from a JSON file (missing keys -> defaults). */ + PluginvalSettings fromJsonFile (const juce::File&); + + //============================================================================== + /** Parses a random seed from "0x..." hex or a decimal integer, preserving the + historical character-set validation. Throws std::invalid_argument if invalid. + */ + std::int64_t parseRandomSeed (const juce::String& raw); + + /** Resolves --disabled-tests: an absolute path is read line-by-line, otherwise + the value is treated as a comma-separated list. + */ + std::vector disabledTestsToList (const juce::String& raw); +} diff --git a/Source/StrictnessInfoPopup.h b/source/StrictnessInfoPopup.h similarity index 100% rename from Source/StrictnessInfoPopup.h rename to source/StrictnessInfoPopup.h diff --git a/Source/TestUtilities.cpp b/source/TestUtilities.cpp similarity index 100% rename from Source/TestUtilities.cpp rename to source/TestUtilities.cpp diff --git a/Source/TestUtilities.h b/source/TestUtilities.h similarity index 100% rename from Source/TestUtilities.h rename to source/TestUtilities.h diff --git a/Source/Validator.cpp b/source/Validator.cpp similarity index 99% rename from Source/Validator.cpp rename to source/Validator.cpp index 529866f8..e9a6fdfa 100644 --- a/Source/Validator.cpp +++ b/source/Validator.cpp @@ -22,7 +22,7 @@ //============================================================================== struct PluginsUnitTestRunner : public juce::UnitTestRunner, - private juce::Thread + private juce::Thread { PluginsUnitTestRunner (std::function logCallback, std::unique_ptr logDestination, juce::int64 timeoutInMs) : Thread ("TimoutThread"), diff --git a/Source/Validator.h b/source/Validator.h similarity index 100% rename from Source/Validator.h rename to source/Validator.h diff --git a/source/acceptance/AcceptanceTest.cpp b/source/acceptance/AcceptanceTest.cpp new file mode 100644 index 00000000..26d67a80 --- /dev/null +++ b/source/acceptance/AcceptanceTest.cpp @@ -0,0 +1,697 @@ +/*============================================================================== + + Copyright 2018 by Tracktion Corporation. + For more information visit www.tracktion.com + + You may also use this code under the terms of the GPL v3 (see + www.gnu.org/licenses). + + pluginval IS PROVIDED "AS IS" WITHOUT ANY WARRANTY, AND ALL WARRANTIES, WHETHER + EXPRESSED OR IMPLIED, INCLUDING MERCHANTABILITY AND FITNESS FOR PURPOSE, ARE + DISCLAIMED. + + ==============================================================================*/ + +#include "AcceptanceTest.h" +#include "../TestUtilities.h" + +#include + +#define XXH_INLINE_ALL +#include + +#include +#include +#include +#include +#include +#include + +namespace acceptance +{ + +//============================================================================== +namespace +{ + juce::String archString() + { + #if defined (__aarch64__) || defined (_M_ARM64) + return "arm64"; + #elif defined (__x86_64__) || defined (_M_X64) + return "x86_64"; + #else + return "unknown"; + #endif + } + + //============================================================================== + /** A fixed-tempo transport whose position advances with the render. Constructed + from the config's playhead block and pointed at the plugin for the duration + of the render; setSamplePosition() is called before each processBlock. */ + class FixedPlayHead : public juce::AudioPlayHead + { + public: + FixedPlayHead (const TestConfig::PlayheadConfig& config, double sr) + : cfg (config), sampleRate (sr) + { + } + + void setSamplePosition (juce::int64 sample) noexcept { currentSample = sample; } + + juce::Optional getPosition() const override + { + const double seconds = (double) currentSample / sampleRate; + + // PPQ is measured in quarter notes; a bar is numerator * (4/denominator) of them. + const double quarterNotesPerBar = cfg.timeSigNumerator * 4.0 / cfg.timeSigDenominator; + const double ppq = cfg.startPpq + seconds * (cfg.bpm / 60.0); + + PositionInfo info; + info.setBpm (cfg.bpm); + info.setTimeSignature (TimeSignature { cfg.timeSigNumerator, cfg.timeSigDenominator }); + info.setTimeInSamples (currentSample); + info.setTimeInSeconds (seconds); + info.setPpqPosition (ppq); + info.setPpqPositionOfLastBarStart (std::floor (ppq / quarterNotesPerBar) * quarterNotesPerBar); + info.setIsPlaying (true); + return info; + } + + private: + const TestConfig::PlayheadConfig cfg; + const double sampleRate; + juce::int64 currentSample = 0; + }; + + //============================================================================== + std::unique_ptr loadPlugin (juce::AudioPluginFormatManager& formatManager, + const juce::String& pathOrID, + double sampleRate, int blockSize) + { + juce::KnownPluginList list; + juce::OwnedArray found; + list.scanAndAddDragAndDroppedFiles (formatManager, juce::StringArray (pathOrID), found); + + if (found.isEmpty()) + throw std::runtime_error (("no plugin found at: " + pathOrID + + " (missing/damaged binary, an incompatible format, or an AU not registered with macOS)").toStdString()); + + juce::String error; + auto instance = std::unique_ptr ( + formatManager.createPluginInstance (*found.getFirst(), sampleRate, blockSize, error)); + + if (instance == nullptr) + throw std::runtime_error (("failed to create plugin instance: " + error).toStdString()); + + return instance; + } + + //============================================================================== + /** Deletes the plugin on the message thread, as hosts are expected to. */ + void deleteOnMessageThread (std::unique_ptr instance) + { + if (juce::MessageManager::getInstance()->isThisTheMessageThread()) + { + instance.reset(); + return; + } + + juce::WaitableEvent deleted; + juce::MessageManager::callAsync ([&] + { + instance.reset(); + deleted.signal(); + }); + deleted.wait(); + } + + /** Owns the plugin for the duration of the render. On the way out (including + when an exception unwinds the render) it detaches the playhead, releases + resources if the plugin was prepared and deletes it on the message thread. */ + struct PluginLifetime + { + ~PluginLifetime() + { + if (instance == nullptr) + return; + + instance->setPlayHead (nullptr); + + if (prepared) + callReleaseResourcesOnMessageThreadIfVST3 (*instance); + + deleteOnMessageThread (std::move (instance)); + } + + std::unique_ptr instance; + bool prepared = false; + }; + + //============================================================================== + void applyState (juce::AudioPluginInstance& instance, const TestConfig& config) + { + // Binary state first (the full plugin state), then the parameter map as + // overrides (see the precedence rule in the spec). + if (const auto stateFile = config.getStateFile(); stateFile != juce::File()) + { + if (! stateFile.existsAsFile()) + throw std::runtime_error (("state.file not found: " + stateFile.getFullPathName()).toStdString()); + + juce::MemoryBlock state; + stateFile.loadFileAsData (state); + callSetStateInformationOnMessageThreadIfVST3 (instance, state); + } + + for (const auto& [key, value] : config.stateParameters) + { + const juce::String name (key); + juce::AudioProcessorParameter* match = nullptr; + + if (name.containsOnly ("0123456789") && name.isNotEmpty()) + { + match = instance.getParameters()[name.getIntValue()]; + } + else + { + // Match the format's parameter ID, then the display name (both + // case-insensitively). Hosted parameters are HostedParameters, whose + // ID is the format's own (e.g. an LV2 symbol), not the JUCE paramID. + for (auto* p : instance.getParameters()) + { + if (auto* hosted = dynamic_cast (p)) + if (hosted->getParameterID().equalsIgnoreCase (name)) + { + match = p; + break; + } + + if (p->getName (512).equalsIgnoreCase (name)) + { + match = p; + break; + } + } + } + + if (match == nullptr) + throw std::runtime_error (("state.parameters: no parameter named or indexed \"" + name + "\"").toStdString()); + + match->setValueNotifyingHost ((float) value); + } + } + + //============================================================================== + struct AudioFileData + { + juce::AudioBuffer buffer; + double sampleRate = 0.0; + }; + + AudioFileData readWav (juce::AudioFormatManager& formatManager, const juce::File& file) + { + std::unique_ptr reader (formatManager.createReaderFor (file)); + + if (reader == nullptr) + throw std::runtime_error (("could not read audio file: " + file.getFullPathName()).toStdString()); + + if (reader->lengthInSamples > std::numeric_limits::max()) + throw std::runtime_error (("audio file is too long: " + file.getFullPathName()).toStdString()); + + const auto numSamples = (int) reader->lengthInSamples; + AudioFileData data { juce::AudioBuffer ((int) reader->numChannels, numSamples), reader->sampleRate }; + + if (! reader->read (&data.buffer, 0, numSamples, 0, true, true)) + throw std::runtime_error (("failed reading audio file: " + file.getFullPathName()).toStdString()); + + return data; + } + + void writeFloatWav (const juce::File& file, const juce::AudioBuffer& buffer, double sampleRate) + { + file.getParentDirectory().createDirectory(); + file.deleteFile(); + + std::unique_ptr stream (file.createOutputStream()); + + if (stream == nullptr) + throw std::runtime_error (("could not open for writing: " + file.getFullPathName()).toStdString()); + + juce::WavAudioFormat wav; + const auto options = juce::AudioFormatWriterOptions() + .withSampleRate (sampleRate) + .withNumChannels (buffer.getNumChannels()) + .withBitsPerSample (32) + .withSampleFormat (juce::AudioFormatWriterOptions::SampleFormat::floatingPoint); + + std::unique_ptr writer (wav.createWriterFor (stream, options)); + + if (writer == nullptr) + throw std::runtime_error (("could not create WAV writer for: " + file.getFullPathName()).toStdString()); + + writer->writeFromAudioSampleBuffer (buffer, 0, buffer.getNumSamples()); + } + + //============================================================================== + /** Builds a single MidiBuffer with events at absolute sample positions. */ + juce::MidiBuffer loadMidi (const juce::File& file, double sampleRate) + { + juce::FileInputStream stream (file); + + if (! stream.openedOk()) + throw std::runtime_error (("could not read MIDI file: " + file.getFullPathName()).toStdString()); + + juce::MidiFile midiFile; + + if (! midiFile.readFrom (stream)) + throw std::runtime_error (("could not parse MIDI file: " + file.getFullPathName()).toStdString()); + + midiFile.convertTimestampTicksToSeconds(); + + juce::MidiBuffer buffer; + + for (int t = 0; t < midiFile.getNumTracks(); ++t) + if (const auto* track = midiFile.getTrack (t)) + for (const auto* event : *track) + { + // Meta events (tempo, track name, end of track...) are file + // structure, not MIDI to send to the plugin. + if (event->message.isMetaEvent()) + continue; + + const auto sample = (int) std::lround (event->message.getTimeStamp() * sampleRate); + buffer.addEvent (event->message, sample); + } + + return buffer; + } + + //============================================================================== + juce::String toHex (XXH64_hash_t hash) + { + return juce::String::toHexString ((juce::int64) hash).paddedLeft ('0', 16); + } + + /** XXH3 of a file's contents, or "missing" if it can't be read. */ + juce::String hashFile (const juce::File& file) + { + juce::MemoryBlock data; + + if (! file.loadFileAsData (data)) + return "missing"; + + return toHex (XXH3_64bits (data.getData(), data.getSize())); + } + + /** A hash of everything that determines the render apart from the plugin + binary itself (which is what's under test). Paths are machine-specific, + so they are left out: the input and state files are hashed by content + instead, and the plugin and reference locations are dropped. */ + juce::String configHash (const TestConfig& config) + { + auto j = nlohmann::ordered_json {}; + to_json (j, config); + j.erase ("plugin"); + j.erase ("reference"); + + if (auto input = j.find ("input"); input != j.end()) + { + if (input->contains ("audio")) (*input)["audio"] = hashFile (config.getInputAudioFile()).toStdString(); + if (input->contains ("midi")) (*input)["midi"] = hashFile (config.getInputMidiFile()).toStdString(); + } + + if (auto state = j.find ("state"); state != j.end() && state->contains ("file")) + (*state)["file"] = hashFile (config.getStateFile()).toStdString(); + + const auto dump = j.dump(); + return toHex (XXH3_64bits (dump.data(), dump.size())); + } +} + +//============================================================================== +RenderedAudio renderPlugin (const TestConfig& config) +{ + const auto sampleRate = config.sampleRate; + const auto blockSize = config.blockSize; // validated by from_json + + juce::AudioPluginFormatManager formatManager; + #if JUCE_VERSION >= 0x08000B + juce::addDefaultFormatsToManager (formatManager); + #else + formatManager.addDefaultFormats(); + #endif + + // Declared before the plugin so it outlives it. + std::unique_ptr playHead; + + PluginLifetime plugin; + plugin.instance = loadPlugin (formatManager, config.getPluginPathOrID(), sampleRate, blockSize); + auto& instance = plugin.instance; + const auto description = instance->getPluginDescription(); + + // 1. State (file then parameter overrides), applied before prepareToPlay. + applyState (*instance, config); + + // 2. Input audio / MIDI (or silence). + juce::AudioFormatManager audioFormats; + audioFormats.registerBasicFormats(); + + juce::AudioBuffer inputAudio; + if (const auto audioFile = config.getInputAudioFile(); audioFile != juce::File()) + { + if (! audioFile.existsAsFile()) + throw std::runtime_error (("input.audio not found: " + audioFile.getFullPathName()).toStdString()); + + auto data = readWav (audioFormats, audioFile); + + if (data.sampleRate != sampleRate) + throw std::runtime_error (("input.audio sample rate (" + juce::String (data.sampleRate) + ") doesn't match sample_rate (" + + juce::String (sampleRate) + "): " + audioFile.getFullPathName()).toStdString()); + + inputAudio = std::move (data.buffer); + } + + juce::MidiBuffer midi; + if (const auto midiFile = config.getInputMidiFile(); midiFile != juce::File()) + { + if (! midiFile.existsAsFile()) + throw std::runtime_error (("input.midi not found: " + midiFile.getFullPathName()).toStdString()); + + midi = loadMidi (midiFile, sampleRate); + } + + // 3. Render length. + int numSamples = 0; + if (config.renderDuration) + { + const auto samples = std::llround (*config.renderDuration * sampleRate); + + if (samples > std::numeric_limits::max()) + throw std::runtime_error ("render_duration * sample_rate is too many samples"); + + numSamples = (int) samples; + } + else if (inputAudio.getNumSamples() > 0) + numSamples = inputAudio.getNumSamples(); + + if (numSamples <= 0) + throw std::runtime_error ("render_duration is required when there is no input.audio"); + + // 4. Optional fixed transport for time-dependent plugins. + if (config.playhead) + { + playHead = std::make_unique (*config.playhead, sampleRate); + instance->setPlayHead (playHead.get()); + } + + // 5. Prepare and render block by block. + callPrepareToPlayOnMessageThreadIfVST3 (*instance, sampleRate, blockSize); + plugin.prepared = true; + + const int numInputChannels = instance->getTotalNumInputChannels(); + const int numOutputChannels = juce::jmax (1, instance->getTotalNumOutputChannels()); + const int channelsRequired = juce::jmax (numInputChannels, numOutputChannels); + + juce::AudioBuffer output (numOutputChannels, numSamples); + output.clear(); + + juce::AudioBuffer block (channelsRequired, blockSize); + + for (int pos = 0; pos < numSamples; pos += blockSize) + { + const int thisBlock = juce::jmin (blockSize, numSamples - pos); + + if (playHead != nullptr) + playHead->setSamplePosition (pos); + + block.clear(); + + // Copy the input audio slice into the block (silence past its end). + for (int c = 0; c < juce::jmin (numInputChannels, inputAudio.getNumChannels()); ++c) + { + const int available = juce::jmax (0, juce::jmin (thisBlock, inputAudio.getNumSamples() - pos)); + if (available > 0) + block.copyFrom (c, 0, inputAudio, c, pos, available); + } + + juce::MidiBuffer blockMidi; + blockMidi.addEvents (midi, pos, thisBlock, -pos); + + juce::AudioBuffer proc (block.getArrayOfWritePointers(), channelsRequired, thisBlock); + instance->processBlock (proc, blockMidi); + + for (int c = 0; c < numOutputChannels; ++c) + output.copyFrom (c, pos, proc, c, 0, thisBlock); + } + + return { std::move (output), sampleRate, blockSize, description }; +} + +//============================================================================== +TestResult runTest (const TestConfig& config, const RunOptions& options) +{ + const auto name = config.getName(); + + try + { + const auto referenceFile = config.getReferenceFile(); + + // Fail fast: without --record-missing, a missing reference is an error + // (e.g. a golden file deleted or mis-pathed in CI must not silently pass). + if (! referenceFile.existsAsFile() && ! options.recordMissing) + { + auto result = TestResult::makeError (name, "reference not found: " + referenceFile.getFullPathName() + + " (run with --record-missing to record it)"); + result.referenceFile = referenceFile; + return result; + } + + const auto rendered = renderPlugin (config); + + TestResult result; + result.name = name; + result.referenceFile = referenceFile; + + // Record mode: no reference yet -> write the float WAV + JSON sidecar. + if (! referenceFile.existsAsFile()) + { + writeFloatWav (referenceFile, rendered.buffer, rendered.sampleRate); + + nlohmann::json manifest; + manifest["plugin"] = { + { "name", rendered.description.name.toStdString() }, + { "manufacturer", rendered.description.manufacturerName.toStdString() }, + { "version", rendered.description.version.toStdString() }, + { "format", rendered.description.pluginFormatName.toStdString() }, + { "uid", rendered.description.createIdentifierString().toStdString() } + }; + manifest["render"] = { + { "sample_rate", rendered.sampleRate }, + { "block_size", rendered.blockSize }, + { "num_channels", rendered.buffer.getNumChannels() }, + { "num_samples", rendered.buffer.getNumSamples() }, + { "length_seconds", rendered.buffer.getNumSamples() / rendered.sampleRate } + }; + manifest["pluginval_version"] = VERSION; + manifest["config_hash"] = configHash (config).toStdString(); + manifest["created_on"] = { + { "os", juce::SystemStats::getOperatingSystemName().toStdString() }, + { "arch", archString().toStdString() }, + { "date", juce::Time::getCurrentTime().toISO8601 (true).toStdString() } + }; + + config.getReferenceSidecarFile().replaceWithText (manifest.dump (2)); + + result.outcome = TestResult::Outcome::referenceCreated; + return result; + } + + // Compare mode: warn (don't fail) if the stored config hash differs. + if (const auto sidecar = config.getReferenceSidecarFile(); sidecar.existsAsFile()) + { + try + { + const auto manifest = nlohmann::json::parse (sidecar.loadFileAsString().toStdString()); + const auto currentHash = configHash (config).toStdString(); + + if (manifest.contains ("config_hash") + && manifest["config_hash"].get() != currentHash) + { + std::cout << " WARNING: reference was recorded from a different config (stale?): " + << sidecar.getFullPathName() << " (recorded " << manifest["config_hash"].get() + << ", current " << currentHash << ")" << std::endl; + } + } + catch (const std::exception&) { /* a malformed sidecar is non-fatal */ } + } + + juce::AudioFormatManager formats; + formats.registerBasicFormats(); + const auto referenceData = readWav (formats, referenceFile); + const auto& reference = referenceData.buffer; + + if (referenceData.sampleRate != rendered.sampleRate) + return TestResult::makeError (name, "reference sample rate (" + juce::String (referenceData.sampleRate) + + ") doesn't match the render (" + juce::String (rendered.sampleRate) + "): " + + referenceFile.getFullPathName()); + + bool allPassed = true; + + const auto comparisonConfig = config.getComparison(); + + for (auto it = comparisonConfig.begin(); it != comparisonConfig.end(); ++it) + { + const juce::String method (it.key()); + auto comparator = createComparator (method); + + if (comparator == nullptr) + return TestResult::makeError (name, "unknown comparison method: " + method); + + const auto result2 = comparator->compare (reference, rendered.buffer, it.value()); + allPassed = allPassed && result2.passed; + result.comparisons.emplace_back (method, result2); + } + + result.outcome = allPassed ? TestResult::Outcome::passed : TestResult::Outcome::failed; + + // Never leave a diff from an earlier failure behind. + config.getDiffFile().deleteFile(); + + if (! allPassed) + { + // Write a diff WAV (output - reference) over the overlapping region. + const int channels = juce::jmin (reference.getNumChannels(), rendered.buffer.getNumChannels()); + const int samples = juce::jmin (reference.getNumSamples(), rendered.buffer.getNumSamples()); + + if (channels > 0 && samples > 0) + { + juce::AudioBuffer diff (channels, samples); + + for (int c = 0; c < channels; ++c) + { + diff.copyFrom (c, 0, rendered.buffer, c, 0, samples); + diff.addFrom (c, 0, reference, c, 0, samples, -1.0f); + } + + const auto diffFile = config.getDiffFile(); + writeFloatWav (diffFile, diff, rendered.sampleRate); + result.diffFile = diffFile; + } + } + + return result; + } + catch (const std::exception& e) + { + return TestResult::makeError (name, e.what()); + } + catch (...) + { + return TestResult::makeError (name, "unknown exception"); + } +} + +//============================================================================== +int runTestFile (const juce::File& configFile, const RunOptions& options) +{ + std::vector configs; + + try + { + configs = TestConfig::loadFromFile (configFile); + } + catch (const std::exception& e) + { + const auto result = TestResult::makeError (configFile.getFileNameWithoutExtension(), e.what()); + reporter::report (result); + return reporter::exitCode (result); + } + + if (configs.empty()) + { + const auto result = TestResult::makeError (configFile.getFileNameWithoutExtension(), "no test configs in file"); + reporter::report (result); + return reporter::exitCode (result); + } + + // v1 runs the first entry only; the parser already accepts arrays for the + // phase-2 multiplexing extension. + if (configs.size() > 1) + std::cout << "Note: " << configs.size() << " configs found; v1 runs the first only." << std::endl; + + const auto result = runTest (configs.front(), options); + reporter::report (result); + return reporter::exitCode (result); +} + +//============================================================================== +struct TestRunner::Watchdog : private juce::Thread +{ + Watchdog (const juce::File& file, juce::int64 timeout) + : Thread ("pluginval test watchdog"), configFile (file), timeoutMs (timeout) + { + startThread (juce::Thread::Priority::low); + } + + ~Watchdog() override + { + markFinished(); + stopThread (5000); + } + + void markFinished() + { + finished.signal(); + } + +private: + const juce::File configFile; + const juce::int64 timeoutMs; + juce::WaitableEvent finished; + + void run() override + { + if (finished.wait ((double) timeoutMs)) + return; + + // The worker is stuck in a plugin call that can't be interrupted, so + // report the failure and end the process. + const auto result = TestResult::makeError (configFile.getFileNameWithoutExtension(), + "timed out after " + juce::RelativeTime::milliseconds (timeoutMs).getDescription()); + reporter::report (result); + std::cout << std::flush; + std::_Exit (reporter::exitCode (result)); + } +}; + +//============================================================================== +TestRunner::TestRunner (const juce::File& file, const RunOptions& opts, std::function callback) + : Thread ("pluginval test"), + configFile (file), + options (opts), + onComplete (std::move (callback)) +{ + jassert (onComplete); + + if (options.timeoutMs > 0) + watchdog = std::make_unique (configFile, options.timeoutMs); + + startThread(); +} + +TestRunner::~TestRunner() +{ + stopThread (5000); + watchdog.reset(); +} + +void TestRunner::run() +{ + const auto exitCode = runTestFile (configFile, options); + + if (watchdog != nullptr) + watchdog->markFinished(); + + juce::MessageManager::callAsync ([callback = onComplete, exitCode] { callback (exitCode); }); +} + +} // namespace acceptance diff --git a/source/acceptance/AcceptanceTest.h b/source/acceptance/AcceptanceTest.h new file mode 100644 index 00000000..65ea6bea --- /dev/null +++ b/source/acceptance/AcceptanceTest.h @@ -0,0 +1,91 @@ +/*============================================================================== + + Copyright 2018 by Tracktion Corporation. + For more information visit www.tracktion.com + + You may also use this code under the terms of the GPL v3 (see + www.gnu.org/licenses). + + pluginval IS PROVIDED "AS IS" WITHOUT ANY WARRANTY, AND ALL WARRANTIES, WHETHER + EXPRESSED OR IMPLIED, INCLUDING MERCHANTABILITY AND FITNESS FOR PURPOSE, ARE + DISCLAIMED. + + ==============================================================================*/ + +#pragma once + +#include "TestConfig.h" +#include "TestReporter.h" + +#include + +#include +#include + +namespace acceptance +{ + +//============================================================================== +/** A rendered buffer plus the metadata needed for the reference manifest. */ +struct RenderedAudio +{ + juce::AudioBuffer buffer; + double sampleRate = 0.0; + int blockSize = 0; + juce::PluginDescription description; +}; + +//============================================================================== +/** Loads the plugin, applies state (file then parameters), feeds the input + (audio / MIDI / silence) and renders a fixed duration into a single buffer. + + Intended to be called off the message thread (see TestRunner): VST3 lifecycle + calls and the plugin's destruction are marshalled onto the message thread, + which must be free to service them. Throws std::runtime_error on any setup / + render failure. +*/ +RenderedAudio renderPlugin (const TestConfig&); + +//============================================================================== +/** Options from the "pluginval test" command line. */ +struct RunOptions +{ + bool recordMissing = false; /**< Record a missing reference and pass, rather than failing. */ + juce::int64 timeoutMs = 30000; /**< Abort the whole run after this long. <= 0 never times out. */ +}; + +//============================================================================== +/** Runs a single resolved config end-to-end: render, then record-or-compare, + producing a TestResult. Never throws - setup failures become an error result. */ +TestResult runTest (const TestConfig&, const RunOptions&); + +/** Loads the config file (first entry only for v1), runs it, reports the result + to stdout and returns the process exit code (0 success, 1 failure). */ +int runTestFile (const juce::File& configFile, const RunOptions&); + +//============================================================================== +/** Runs runTestFile() on a background thread so the message thread stays free + to service the plugin (VST3 lifecycle calls, async creation, posted work). + + onComplete is called on the message thread with the exit code. If the run + exceeds the timeout, a watchdog reports the failure and terminates the + process with exit code 1, since a hung plugin call can't be interrupted. +*/ +class TestRunner : private juce::Thread +{ +public: + TestRunner (const juce::File& configFile, const RunOptions&, std::function onComplete); + ~TestRunner() override; + +private: + struct Watchdog; + + const juce::File configFile; + const RunOptions options; + std::function onComplete; + std::unique_ptr watchdog; + + void run() override; +}; + +} // namespace acceptance diff --git a/source/acceptance/ReferenceComparator.cpp b/source/acceptance/ReferenceComparator.cpp new file mode 100644 index 00000000..a036898f --- /dev/null +++ b/source/acceptance/ReferenceComparator.cpp @@ -0,0 +1,134 @@ +/*============================================================================== + + Copyright 2018 by Tracktion Corporation. + For more information visit www.tracktion.com + + You may also use this code under the terms of the GPL v3 (see + www.gnu.org/licenses). + + pluginval IS PROVIDED "AS IS" WITHOUT ANY WARRANTY, AND ALL WARRANTIES, WHETHER + EXPRESSED OR IMPLIED, INCLUDING MERCHANTABILITY AND FITNESS FOR PURPOSE, ARE + DISCLAIMED. + + ==============================================================================*/ + +#include "ReferenceComparator.h" + +#include +#include + +namespace acceptance +{ + +//============================================================================== +/** + Per-sample absolute-difference comparator. + + config is the value under the "sample" key: a bare number giving the + tolerance (0 == bit-exact), or an object { "tolerance": }. Channel + and length mismatches fail outright, as does any non-finite (NaN / inf) + difference. +*/ +struct SampleComparator : public Comparator +{ + juce::String getName() const override { return "sample"; } + + static double toleranceFrom (const nlohmann::ordered_json& config) + { + if (config.is_number()) + return config.get(); + + if (config.is_object()) + if (auto t = config.find ("tolerance"); t != config.end() && t->is_number()) + return t->get(); + + return 1.0 / 32768.0; + } + + ComparisonResult compare (const juce::AudioBuffer& reference, + const juce::AudioBuffer& output, + const nlohmann::ordered_json& config) override + { + const double tolerance = toleranceFrom (config); + + ComparisonResult r; + r.details["tolerance"] = tolerance; + + if (reference.getNumChannels() != output.getNumChannels() + || reference.getNumSamples() != output.getNumSamples()) + { + r.passed = false; + r.summary = "size mismatch: reference " + + juce::String (reference.getNumChannels()) + "ch x " + juce::String (reference.getNumSamples()) + " samples, " + + "output " + juce::String (output.getNumChannels()) + "ch x " + juce::String (output.getNumSamples()) + " samples"; + r.details["reference_channels"] = reference.getNumChannels(); + r.details["reference_samples"] = reference.getNumSamples(); + r.details["output_channels"] = output.getNumChannels(); + r.details["output_samples"] = output.getNumSamples(); + return r; + } + + double maxAbsDiff = 0.0; + bool nonFinite = false; + int firstFailChannel = -1, firstFailSample = -1; + + for (int c = 0; c < reference.getNumChannels(); ++c) + { + const auto* ref = reference.getReadPointer (c); + const auto* out = output.getReadPointer (c); + + for (int s = 0; s < reference.getNumSamples(); ++s) + { + const double diff = std::abs ((double) out[s] - (double) ref[s]); + + // NaN compares false both ways, so test for failure with + // ! (diff <= tolerance) and track non-finite diffs separately. + if (! std::isfinite (diff)) + nonFinite = true; + else if (diff > maxAbsDiff) + maxAbsDiff = diff; + + if (! (diff <= tolerance) && firstFailChannel < 0) + { + firstFailChannel = c; + firstFailSample = s; + } + } + } + + r.score = nonFinite ? std::numeric_limits::infinity() : maxAbsDiff; + r.passed = firstFailChannel < 0; + r.details["max_abs_diff"] = maxAbsDiff; + r.details["non_finite"] = nonFinite; + + if (r.passed) + { + r.summary = "max abs diff " + juce::String (maxAbsDiff) + " <= tolerance " + juce::String (tolerance); + } + else + { + const auto firstFail = " (first at channel " + juce::String (firstFailChannel) + ", sample " + juce::String (firstFailSample) + ")"; + + if (nonFinite) + r.summary = "non-finite (NaN / inf) difference; finite max abs diff " + juce::String (maxAbsDiff) + firstFail; + else + r.summary = "max abs diff " + juce::String (maxAbsDiff) + " > tolerance " + juce::String (tolerance) + firstFail; + r.details["first_fail_channel"] = firstFailChannel; + r.details["first_fail_sample"] = firstFailSample; + } + + return r; + } +}; + +//============================================================================== +std::unique_ptr createComparator (const juce::String& name) +{ + if (name == "sample") + return std::make_unique(); + + // Future: "peakrms", "spectrum", "crosscorr", "fingerprint" register here. + return {}; +} + +} // namespace acceptance diff --git a/source/acceptance/ReferenceComparator.h b/source/acceptance/ReferenceComparator.h new file mode 100644 index 00000000..4f169ace --- /dev/null +++ b/source/acceptance/ReferenceComparator.h @@ -0,0 +1,60 @@ +/*============================================================================== + + Copyright 2018 by Tracktion Corporation. + For more information visit www.tracktion.com + + You may also use this code under the terms of the GPL v3 (see + www.gnu.org/licenses). + + pluginval IS PROVIDED "AS IS" WITHOUT ANY WARRANTY, AND ALL WARRANTIES, WHETHER + EXPRESSED OR IMPLIED, INCLUDING MERCHANTABILITY AND FITNESS FOR PURPOSE, ARE + DISCLAIMED. + + ==============================================================================*/ + +#pragma once + +#include +#include + +#include + +namespace acceptance +{ + +//============================================================================== +/** The outcome of a single comparator run. */ +struct ComparisonResult +{ + bool passed = false; + double score = 0.0; /**< Method-specific metric, e.g. max abs diff. */ + juce::String summary; /**< Human-readable one-liner. */ + nlohmann::json details; /**< Structured detail for the machine-readable report. */ +}; + +//============================================================================== +/** + Pluggable comparison method. v1 ships only "sample"; further methods + (spectrum, crosscorr, fingerprint) register later with no changes to config + parsing or the runner. +*/ +struct Comparator +{ + virtual ~Comparator() = default; + + /** The method name, e.g. "sample". */ + virtual juce::String getName() const = 0; + + /** Compares a freshly rendered buffer against the reference. config is the + value sitting under this comparator's key in the config's "comparison" + map (a bare number for "sample", or an object for richer methods). */ + virtual ComparisonResult compare (const juce::AudioBuffer& reference, + const juce::AudioBuffer& output, + const nlohmann::ordered_json& config) = 0; +}; + +//============================================================================== +/** Creates a comparator by name, or nullptr if the name is unknown. */ +std::unique_ptr createComparator (const juce::String& name); + +} // namespace acceptance diff --git a/source/acceptance/TestConfig.cpp b/source/acceptance/TestConfig.cpp new file mode 100644 index 00000000..bdb25d0a --- /dev/null +++ b/source/acceptance/TestConfig.cpp @@ -0,0 +1,287 @@ +/*============================================================================== + + Copyright 2018 by Tracktion Corporation. + For more information visit www.tracktion.com + + You may also use this code under the terms of the GPL v3 (see + www.gnu.org/licenses). + + pluginval IS PROVIDED "AS IS" WITHOUT ANY WARRANTY, AND ALL WARRANTIES, WHETHER + EXPRESSED OR IMPLIED, INCLUDING MERCHANTABILITY AND FITNESS FOR PURPOSE, ARE + DISCLAIMED. + + ==============================================================================*/ + +#include "TestConfig.h" + +#include +#include +#include + +namespace acceptance +{ + +//============================================================================== +namespace +{ + /** The default comparison when none is supplied: one 16-bit LSB. */ + nlohmann::ordered_json defaultComparison() + { + return nlohmann::ordered_json { { "sample", 1.0 / 32768.0 } }; + } + + /** Reads a string member from a (possibly absent) nested object. */ + std::string getNestedString (const nlohmann::ordered_json& j, const char* outer, const char* inner) + { + if (auto o = j.find (outer); o != j.end() && o->is_object()) + if (auto i = o->find (inner); i != o->end() && i->is_string()) + return i->get(); + + return {}; + } +} + +//============================================================================== +void to_json (nlohmann::ordered_json& j, const TestConfig& c) +{ + j = nlohmann::ordered_json::object(); + j["name"] = c.name; + j["plugin"] = c.plugin; + + if (! c.inputAudio.empty() || ! c.inputMidi.empty()) + { + auto input = nlohmann::ordered_json::object(); + if (! c.inputAudio.empty()) input["audio"] = c.inputAudio; + if (! c.inputMidi.empty()) input["midi"] = c.inputMidi; + j["input"] = input; + } + + if (! c.reference.empty()) + j["reference"] = c.reference; + + if (! c.stateFile.empty() || ! c.stateParameters.empty()) + { + auto state = nlohmann::ordered_json::object(); + if (! c.stateFile.empty()) state["file"] = c.stateFile; + + if (! c.stateParameters.empty()) + { + auto params = nlohmann::ordered_json::object(); + + for (const auto& [key, value] : c.stateParameters) + params[key] = value; + + state["parameters"] = params; + } + + j["state"] = state; + } + + j["sample_rate"] = c.sampleRate; + j["block_size"] = c.blockSize; + + if (c.renderDuration) + j["render_duration"] = *c.renderDuration; + + if (! c.comparison.is_null()) + j["comparison"] = c.comparison; + + if (c.playhead) + { + j["playhead"] = { + { "bpm", c.playhead->bpm }, + { "time_signature", { { "numerator", c.playhead->timeSigNumerator }, + { "denominator", c.playhead->timeSigDenominator } } }, + { "start_ppq", c.playhead->startPpq } + }; + } +} + +void from_json (const nlohmann::ordered_json& j, TestConfig& c) +{ + const auto fail = [] (const std::string& message) { throw std::runtime_error (message); }; + + c.name = j.value ("name", std::string()); + c.plugin = j.value ("plugin", std::string()); + c.reference = j.value ("reference", std::string()); + + c.inputAudio = getNestedString (j, "input", "audio"); + c.inputMidi = getNestedString (j, "input", "midi"); + + c.stateFile = getNestedString (j, "state", "file"); + + if (auto state = j.find ("state"); state != j.end() && state->is_object()) + if (auto params = state->find ("parameters"); params != state->end() && params->is_object()) + for (const auto& [key, value] : params->items()) + c.stateParameters.emplace_back (key, value.get()); + + c.sampleRate = j.value ("sample_rate", 44100.0); + c.blockSize = j.value ("block_size", 512); + + if (auto d = j.find ("render_duration"); d != j.end() && ! d->is_null()) + { + if (! d->is_number()) + fail ("render_duration must be a number of seconds"); + + c.renderDuration = d->get(); + } + + if (auto comp = j.find ("comparison"); comp != j.end() && ! comp->is_null()) + { + if (! comp->is_object() || comp->empty()) + fail ("comparison must be a non-empty object of method -> settings (omit it for the default)"); + + c.comparison = *comp; + } + + // Validation. + if (c.plugin.empty()) + fail ("plugin is required"); + + if (c.name.empty() && c.reference.empty()) + fail ("name or reference is required"); + + if (! std::isfinite (c.sampleRate) || c.sampleRate <= 0.0) + fail ("sample_rate must be a positive number"); + + if (c.blockSize < 1 || c.blockSize > 65536) + fail ("block_size must be between 1 and 65536"); + + if (c.renderDuration) + { + const auto duration = *c.renderDuration; + + if (! std::isfinite (duration) || duration <= 0.0) + fail ("render_duration must be a positive number of seconds"); + + if (duration * c.sampleRate > (double) std::numeric_limits::max()) + fail ("render_duration * sample_rate is too many samples"); + } + + if (auto ph = j.find ("playhead"); ph != j.end() && ph->is_object()) + { + TestConfig::PlayheadConfig p; + p.bpm = ph->value ("bpm", 120.0); + p.startPpq = ph->value ("start_ppq", 0.0); + + if (auto ts = ph->find ("time_signature"); ts != ph->end() && ts->is_object()) + { + p.timeSigNumerator = ts->value ("numerator", 4); + p.timeSigDenominator = ts->value ("denominator", 4); + } + + if (p.timeSigNumerator <= 0 || p.timeSigDenominator <= 0) + throw std::runtime_error ("playhead.time_signature numerator and denominator must be positive"); + + c.playhead = p; + } +} + +//============================================================================== +nlohmann::ordered_json TestConfig::getComparison() const +{ + return comparison.is_null() ? defaultComparison() : comparison; +} + +juce::String TestConfig::getPluginPathOrID() const +{ + const juce::String raw (plugin); + + // Resolve relative/home paths against the working directory; leave absolute + // paths and bare component IDs (no '.' or '~') untouched. Mirrors + // settings_parser::resolvePluginPath. + if (raw.contains ("~") || raw.contains (".")) + return juce::File::getCurrentWorkingDirectory().getChildFile (raw).getFullPathName(); + + return raw; +} + +juce::File TestConfig::resolveAgainstConfigDir (const std::string& path) const +{ + if (path.empty()) + return {}; + + const juce::String s (path); + + if (juce::File::isAbsolutePath (s)) + return juce::File (s); + + const auto base = configDir == juce::File() ? juce::File::getCurrentWorkingDirectory() : configDir; + return base.getChildFile (s); +} + +juce::String TestConfig::getName() const +{ + if (! name.empty()) + return juce::String (name); + + return getReferenceFile().getFileNameWithoutExtension(); +} + +juce::File TestConfig::getReferenceFile() const +{ + if (! reference.empty()) + return resolveAgainstConfigDir (reference); + + const auto base = configDir == juce::File() ? juce::File::getCurrentWorkingDirectory() : configDir; + return base.getChildFile (juce::String (name) + ".wav"); +} + +juce::File TestConfig::getReferenceSidecarFile() const +{ + const auto ref = getReferenceFile(); + return ref.getSiblingFile (ref.getFileName() + ".json"); +} + +juce::File TestConfig::getDiffFile() const +{ + const auto ref = getReferenceFile(); + return ref.getSiblingFile (ref.getFileNameWithoutExtension() + "-diff.wav"); +} + +juce::File TestConfig::getInputAudioFile() const { return resolveAgainstConfigDir (inputAudio); } +juce::File TestConfig::getInputMidiFile() const { return resolveAgainstConfigDir (inputMidi); } +juce::File TestConfig::getStateFile() const { return resolveAgainstConfigDir (stateFile); } + +//============================================================================== +std::vector TestConfig::loadFromFile (const juce::File& file) +{ + if (! file.existsAsFile()) + throw std::runtime_error (("test config not found: " + file.getFullPathName()).toStdString()); + + // ordered_json keeps object keys in file order (state.parameters is applied + // in that order). + nlohmann::ordered_json j; + + try + { + j = nlohmann::ordered_json::parse (file.loadFileAsString().toStdString()); + } + catch (const std::exception& e) + { + throw std::runtime_error (("failed to parse test config " + file.getFullPathName() + ": " + e.what()).toStdString()); + } + + std::vector configs; + + const auto addOne = [&] (const nlohmann::ordered_json& entry) + { + auto c = entry.get(); + c.configDir = file.getParentDirectory(); + configs.push_back (std::move (c)); + }; + + if (j.is_array()) + { + for (const auto& entry : j) + addOne (entry); + } + else + { + addOne (j); + } + + return configs; +} + +} // namespace acceptance diff --git a/source/acceptance/TestConfig.h b/source/acceptance/TestConfig.h new file mode 100644 index 00000000..556a5823 --- /dev/null +++ b/source/acceptance/TestConfig.h @@ -0,0 +1,119 @@ +/*============================================================================== + + Copyright 2018 by Tracktion Corporation. + For more information visit www.tracktion.com + + You may also use this code under the terms of the GPL v3 (see + www.gnu.org/licenses). + + pluginval IS PROVIDED "AS IS" WITHOUT ANY WARRANTY, AND ALL WARRANTIES, WHETHER + EXPRESSED OR IMPLIED, INCLUDING MERCHANTABILITY AND FITNESS FOR PURPOSE, ARE + DISCLAIMED. + + ==============================================================================*/ + +#pragma once + +#include +#include + +#include +#include +#include +#include + +namespace acceptance +{ + +//============================================================================== +/** + The definition of a single acceptance test. + + This is a plain, std-typed struct deserialised from the positional + of the "pluginval test" command. JSON keys are snake_case + (e.g. sample_rate); the C++ members stay JUCE-style camelCase, so the mapping + is done explicitly in the per-struct to_json / from_json (see TestConfig.cpp) + rather than the bare NLOHMANN macro. It uses nlohmann::ordered_json so object + keys keep their file order (state.parameters is applied in that order). + + This config is completely independent of PluginvalSettings / the --config + settings layering used by the validate command. +*/ +struct TestConfig +{ + /** A fixed transport supplied to the plugin during the render, for + time-dependent plugins (tempo-synced LFOs, arpeggiators, ...). The tempo + and time signature are constant; the position advances with the render. */ + struct PlayheadConfig + { + double bpm = 120.0; + int timeSigNumerator = 4; + int timeSigDenominator = 4; + double startPpq = 0.0; /**< Transport position (in quarter notes) at sample 0. */ + }; + + std::string name; /**< Labels results; derives the default reference path. */ + std::string plugin; /**< Plugin path or AU id. Required. */ + std::string inputAudio; /**< input.audio: path to an input audio file, or empty for silence. */ + std::string inputMidi; /**< input.midi: path to a .mid file, or empty for none. */ + std::string reference; /**< The golden file. Empty -> derived from name. */ + std::string stateFile; /**< state.file: binary getStateInformation blob. */ + std::vector> stateParameters; /**< state.parameters: name-or-index -> normalised value, in file order. */ + double sampleRate = 44100.0; + int blockSize = 512; + std::optional renderDuration; /**< Seconds. Unset -> derive from the input length. */ + nlohmann::ordered_json comparison; /**< Map of comparator name -> sub-config. Null -> default. */ + std::optional playhead; /**< Fixed transport. Unset -> no playhead supplied to the plugin. */ + + //============================================================================== + /** The directory the config was loaded from. Relative reference / input / + state paths resolve against this; the plugin path resolves against the + working directory (per the spec). Not part of the JSON. */ + juce::File configDir; + + //============================================================================== + /** Returns the resolved comparison map, substituting the default + ({ "sample": 1/32768 }) when none was supplied. */ + nlohmann::ordered_json getComparison() const; + + /** The plugin path or AU id, resolved against the working directory. */ + juce::String getPluginPathOrID() const; + + /** The golden reference file (explicit, or /.wav). */ + juce::File getReferenceFile() const; + + /** The reference's JSON sidecar (.json). */ + juce::File getReferenceSidecarFile() const; + + /** The diff WAV written next to the reference on a comparison failure. */ + juce::File getDiffFile() const; + + /** Resolved input audio file, or an empty File if none. */ + juce::File getInputAudioFile() const; + + /** Resolved input MIDI file, or an empty File if none. */ + juce::File getInputMidiFile() const; + + /** Resolved state file, or an empty File if none. */ + juce::File getStateFile() const; + + /** The effective test name (explicit, or the reference's basename). */ + juce::String getName() const; + + //============================================================================== + /** Loads one or more configs from a JSON file. The top level may be a single + object or an array of objects (phase 2 multiplexing); v1 callers use the + first entry. Throws std::runtime_error on a load / parse error. */ + static std::vector loadFromFile (const juce::File&); + +private: + juce::File resolveAgainstConfigDir (const std::string& path) const; +}; + +//============================================================================== +void to_json (nlohmann::ordered_json&, const TestConfig&); + +/** Throws std::runtime_error (or a nlohmann type error) on an invalid config. */ +void from_json (const nlohmann::ordered_json&, TestConfig&); + +} // namespace acceptance diff --git a/source/acceptance/TestReporter.cpp b/source/acceptance/TestReporter.cpp new file mode 100644 index 00000000..a95c100e --- /dev/null +++ b/source/acceptance/TestReporter.cpp @@ -0,0 +1,110 @@ +/*============================================================================== + + Copyright 2018 by Tracktion Corporation. + For more information visit www.tracktion.com + + You may also use this code under the terms of the GPL v3 (see + www.gnu.org/licenses). + + pluginval IS PROVIDED "AS IS" WITHOUT ANY WARRANTY, AND ALL WARRANTIES, WHETHER + EXPRESSED OR IMPLIED, INCLUDING MERCHANTABILITY AND FITNESS FOR PURPOSE, ARE + DISCLAIMED. + + ==============================================================================*/ + +#include "TestReporter.h" + +#include + +namespace acceptance::reporter +{ + +//============================================================================== +static const char* toString (TestResult::Outcome o) +{ + switch (o) + { + case TestResult::Outcome::referenceCreated: return "reference-created"; + case TestResult::Outcome::passed: return "passed"; + case TestResult::Outcome::failed: return "failed"; + case TestResult::Outcome::error: return "error"; + } + + return "error"; +} + +nlohmann::json toJson (const TestResult& r) +{ + nlohmann::json j; + j["name"] = r.name.toStdString(); + j["outcome"] = toString (r.outcome); + + if (r.message.isNotEmpty()) + j["message"] = r.message.toStdString(); + + if (r.referenceFile != juce::File()) + j["reference"] = r.referenceFile.getFullPathName().toStdString(); + + if (r.diffFile != juce::File()) + j["diff"] = r.diffFile.getFullPathName().toStdString(); + + if (! r.comparisons.empty()) + { + auto comparisons = nlohmann::json::object(); + + for (const auto& [method, result] : r.comparisons) + { + nlohmann::json c; + c["passed"] = result.passed; + c["score"] = result.score; + c["summary"] = result.summary.toStdString(); + if (! result.details.is_null()) + c["details"] = result.details; + + comparisons[method.toStdString()] = c; + } + + j["comparisons"] = comparisons; + } + + return j; +} + +void report (const TestResult& r) +{ + switch (r.outcome) + { + case TestResult::Outcome::referenceCreated: + std::cout << "Reference created: " << r.referenceFile.getFullPathName() << std::endl; + break; + + case TestResult::Outcome::passed: + std::cout << "PASSED: " << r.name << std::endl; + break; + + case TestResult::Outcome::failed: + std::cout << "*** FAILED: " << r.name << std::endl; + break; + + case TestResult::Outcome::error: + std::cout << "*** ERROR: " << r.name << ": " << r.message << std::endl; + break; + } + + for (const auto& [method, result] : r.comparisons) + std::cout << " [" << method << "] " << (result.passed ? "ok" : "FAIL") << ": " << result.summary << std::endl; + + if (r.diffFile != juce::File()) + std::cout << " diff written to: " << r.diffFile.getFullPathName() << std::endl; + + // The structured result for machine consumers. + std::cout << toJson (r).dump (2) << std::endl; +} + +int exitCode (const TestResult& r) +{ + return (r.outcome == TestResult::Outcome::referenceCreated + || r.outcome == TestResult::Outcome::passed) ? 0 : 1; +} + +} // namespace acceptance::reporter diff --git a/source/acceptance/TestReporter.h b/source/acceptance/TestReporter.h new file mode 100644 index 00000000..10266b77 --- /dev/null +++ b/source/acceptance/TestReporter.h @@ -0,0 +1,73 @@ +/*============================================================================== + + Copyright 2018 by Tracktion Corporation. + For more information visit www.tracktion.com + + You may also use this code under the terms of the GPL v3 (see + www.gnu.org/licenses). + + pluginval IS PROVIDED "AS IS" WITHOUT ANY WARRANTY, AND ALL WARRANTIES, WHETHER + EXPRESSED OR IMPLIED, INCLUDING MERCHANTABILITY AND FITNESS FOR PURPOSE, ARE + DISCLAIMED. + + ==============================================================================*/ + +#pragma once + +#include "ReferenceComparator.h" + +#include +#include + +#include +#include + +namespace acceptance +{ + +//============================================================================== +/** The result of running one acceptance test. */ +struct TestResult +{ + enum class Outcome + { + referenceCreated, /**< No reference existed, so one was recorded. Treated as success. */ + passed, /**< Reference existed and every comparator passed. */ + failed, /**< Reference existed and at least one comparator failed. */ + error /**< Setup failed (plugin load, render, IO, bad config, ...). */ + }; + + Outcome outcome = Outcome::error; + juce::String name; + juce::String message; /**< Error text, or an overall summary. */ + + std::vector> comparisons; /**< method name -> result. */ + + juce::File referenceFile; + juce::File diffFile; /**< Written only on failure. */ + + //============================================================================== + static TestResult makeError (const juce::String& name, const juce::String& message) + { + TestResult r; + r.outcome = Outcome::error; + r.name = name; + r.message = message; + return r; + } +}; + +//============================================================================== +namespace reporter +{ + /** Structured, machine-readable representation of a result. */ + nlohmann::json toJson (const TestResult&); + + /** Prints a human-readable summary followed by the JSON to stdout. */ + void report (const TestResult&); + + /** 0 for referenceCreated / passed, 1 for failed / error. */ + int exitCode (const TestResult&); +} + +} // namespace acceptance diff --git a/Source/binarydata/icon.png b/source/binarydata/icon.png similarity index 100% rename from Source/binarydata/icon.png rename to source/binarydata/icon.png diff --git a/Source/binarydata/icon.svg b/source/binarydata/icon.svg similarity index 100% rename from Source/binarydata/icon.svg rename to source/binarydata/icon.svg diff --git a/source/native/pluginval_native_posix.h b/source/native/pluginval_native_posix.h new file mode 100644 index 00000000..1e5cbd40 --- /dev/null +++ b/source/native/pluginval_native_posix.h @@ -0,0 +1,125 @@ +/*============================================================================== + + Copyright 2018 by Tracktion Corporation. + For more information visit www.tracktion.com + + You may also use this code under the terms of the GPL v3 (see + www.gnu.org/licenses). + + pluginval IS PROVIDED "AS IS" WITHOUT ANY WARRANTY, AND ALL WARRANTIES, WHETHER + EXPRESSED OR IMPLIED, INCLUDING MERCHANTABILITY AND FITNESS FOR PURPOSE, ARE + DISCLAIMED. + + ==============================================================================*/ + +#pragma once + +#if __has_include() + +#include +#include +#include +#include +#include +#include + +#ifdef __printflike + __printflike(2, 3) +#endif + +/** Writes a string to a file descriptor in a async-signal-safe way. I.e. no allocations etc. */ +inline void writeAsyncSignalSafe (int fd, const char* fmt, ...) +{ + char buf[1024]; + + va_list args; + va_start (args, fmt); + // Warning: printf is not 100% async-signal-safe, but should be ok for locale-independent arguments like + // integers, strings, hex... floating point is locale-dependent and not safe to use here. + vsnprintf (buf, sizeof (buf), fmt, args); + va_end (args); + + auto len = strlen (buf); + [[ maybe_unused]] auto r = write (STDERR_FILENO, buf, len); + + if (fd != -1) + [[ maybe_unused]] auto r2 = write (fd, buf, len); +} + +/** Writes the current stack trace and images to a given filepath and stderr. +*/ +inline void writeStackTrace (const char* filePath) +{ + // On Linux & Mac this is a signal handler, and therefore only "async-signal-safe" functions should be used. + // This means nothing that uses malloc (juce::File, juce::String, std::string, std::vector etc.) or buffered I/O. + int fd = open (filePath, O_RDWR | O_CREAT | O_TRUNC, 00644); + + // Write stack traces and images + { + // Recreate the output of backtrace_symbols(), which cannot be used in a + // signal handler because it uses malloc + constexpr int maxNumStacks = 128; + void *stacktrace[maxNumStacks]{}; + int stackCount = backtrace(stacktrace, maxNumStacks); + + constexpr int maxNumImages = 64; + const void *imageAddresses[maxNumImages]{}; + const char *imageNames[maxNumImages]{}; + int imageCount = 0; + + for (int i = 0; i < stackCount; ++i) + { + Dl_info info{}; + + // Warning: dladdr can deadlock under rare conditions on macOS - if dyld + // is adding an image to its list + if (! dladdr (stacktrace[i], &info)) + { + writeAsyncSignalSafe (fd, "%-3d %-35s %p\n", i, "", stacktrace[i]); + continue; + } + + const char* imageName = info.dli_fname ? strrchr (info.dli_fname, '/') : nullptr; + + if (imageName) + { + ++imageName; + + auto it = std::find (std::begin (imageAddresses), std::end (imageAddresses), info.dli_fbase); + + if (it == std::end (imageAddresses) && imageCount < maxNumImages) + { + imageAddresses[imageCount] = info.dli_fbase; + imageNames[imageCount] = imageName; + ++imageCount; + } + } + + if (info.dli_saddr) + { + ptrdiff_t offset = static_cast (stacktrace[i]) - static_cast (info.dli_saddr); + writeAsyncSignalSafe (fd, "%-3d %-35s %p %s + %ld\n", i, imageName, stacktrace[i], info.dli_sname, + offset); + } + else + { + writeAsyncSignalSafe (fd, "%-3d %-35s %p\n", i, imageName, stacktrace[i]); + } + } + + if (imageCount > 0) + { + writeAsyncSignalSafe (fd, "\nBinary Images:"); + + for (int i = 0; i < imageCount; i++) + writeAsyncSignalSafe (fd, "\n%p %s", imageAddresses[i], imageNames[i]); + + writeAsyncSignalSafe (fd, "\n"); + } + } + + if (fd != -1) + close (fd); +} + + #endif //__has_include() \ No newline at end of file diff --git a/Source/tests/BasicTests.cpp b/source/tests/BasicTests.cpp similarity index 100% rename from Source/tests/BasicTests.cpp rename to source/tests/BasicTests.cpp diff --git a/Source/tests/BusTests.cpp b/source/tests/BusTests.cpp similarity index 100% rename from Source/tests/BusTests.cpp rename to source/tests/BusTests.cpp diff --git a/Source/tests/EditorTests.cpp b/source/tests/EditorTests.cpp similarity index 97% rename from Source/tests/EditorTests.cpp rename to source/tests/EditorTests.cpp index 25fc7c41..8eb66345 100644 --- a/Source/tests/EditorTests.cpp +++ b/source/tests/EditorTests.cpp @@ -63,7 +63,7 @@ struct EditorDPITest : public PluginTest { if (instance.hasEditor()) { - ScopedDPIAwarenessDisabler scopedDPIAwarenessDisabler; + juce::ScopedDPIAwarenessDisabler scopedDPIAwarenessDisabler; { ut.logMessage ("Testing opening Editor with DPI Awareness disabled"); diff --git a/Source/tests/ExtremeTests.cpp b/source/tests/ExtremeTests.cpp similarity index 100% rename from Source/tests/ExtremeTests.cpp rename to source/tests/ExtremeTests.cpp diff --git a/Source/tests/LocaleTest.cpp b/source/tests/LocaleTest.cpp similarity index 100% rename from Source/tests/LocaleTest.cpp rename to source/tests/LocaleTest.cpp diff --git a/Source/tests/ParameterFuzzTests.cpp b/source/tests/ParameterFuzzTests.cpp similarity index 100% rename from Source/tests/ParameterFuzzTests.cpp rename to source/tests/ParameterFuzzTests.cpp diff --git a/Source/vst3validator/VST3ValidatorRunner.cpp b/source/vst3validator/VST3ValidatorRunner.cpp similarity index 100% rename from Source/vst3validator/VST3ValidatorRunner.cpp rename to source/vst3validator/VST3ValidatorRunner.cpp diff --git a/Source/vst3validator/VST3ValidatorRunner.h b/source/vst3validator/VST3ValidatorRunner.h similarity index 100% rename from Source/vst3validator/VST3ValidatorRunner.h rename to source/vst3validator/VST3ValidatorRunner.h diff --git a/tests/AddPluginvalTests.cmake b/tests/AddPluginvalTests.cmake index c4850712..133a9775 100644 --- a/tests/AddPluginvalTests.cmake +++ b/tests/AddPluginvalTests.cmake @@ -81,13 +81,14 @@ function (add_pluginval_tests pluginTarget) add_test (NAME "${test_name}" COMMAND "${PLUGINVAL_PROGRAM}" + validate --strictness-level "${PLUGINVAL_STRICTNESS}" --sample-rates "${sample_rates}" --block-sizes "${block_sizes}" --repeat "${PLUGINVAL_REPEATS}" --randomise - --validate "${plugin_artefact}" ${log_dir_arg} + "${plugin_artefact}" ) set_tests_properties ( diff --git a/tests/acceptance/Acceptance testing design.md b/tests/acceptance/Acceptance testing design.md new file mode 100644 index 00000000..836a8051 --- /dev/null +++ b/tests/acceptance/Acceptance testing design.md @@ -0,0 +1,412 @@ +# Acceptance testing (design) + +> Status: **Phase 1 implemented.** The `pluginval test ` mode +> described here is built (`source/acceptance/`, wired into the CLI dispatcher) +> with the `sample` comparator, record-or-compare, float-WAV + JSON-sidecar +> references, a dogfood tone-generator plugin (`tests/test_plugins/tone_generator/`, +> behind `PLUGINVAL_BUILD_TEST_PLUGINS`) and CTest self-tests +> (`tests/acceptance/`). The Phase 2 items in §10 remain stubs/notes. This +> document is the reference spec for both the implementation and future extension. + +## 1. Overview + +The existing `validate` mode is a graded **unit-test suite** (`PluginTests : +juce::UnitTest`, self-registering `PluginTest` instances, pass/fail via +`expect`). It answers "does this plugin conform to the host API and behave +safely?" + +Acceptance testing answers a different question: **"does this plugin produce the +output I expect for a known input and state?"** It is a deterministic +*render + golden-file comparison*: + +1. Load a plugin, apply a known state / parameter set. +2. Feed a known input (audio and/or MIDI, or silence). +3. Render a fixed duration of audio. +4. If no reference exists, **record** one. If a reference exists, **compare** + the freshly rendered output against it and emit a pass/fail verdict. + +Because it is a fundamentally different activity from `validate`, it is built as +a **parallel subsystem** rather than as another `PluginTest`. It reuses the +plugin-loading, lifecycle, and render infrastructure but has its own CLI +command, config schema, and result model. + +### Scope and honest caveats + +- Acceptance testing is only meaningful for plugins that are **deterministic** + given a fixed input + state. A plugin with free-running internal randomness + cannot be golden-tested reliably. The planned playhead / seed controls help, + but cannot make a non-deterministic plugin deterministic. +- The **cross-platform / cross-version portability** goal (matching a reference + produced on a different OS, CPU, or plugin version) is realistic only with + tolerant comparison methods. Exact / per-sample comparison rarely survives + SIMD and platform floating-point differences. This is precisely why the + comparator is a pluggable abstraction (see §5): spectrum, cross-correlation + and fingerprint methods are what make portable references viable. + +## 2. Command-line interface + +A new subcommand sits alongside the existing verbs: + +``` +pluginval test +``` + +- `` is the **positional** acceptance-test definition (see §4). It + is parsed by its own loader. +- If the config's reference file does not exist, the command fails, unless + `--record-missing` (or `RECORD_MISSING=1`) is given, in which case it is + **created** (record mode) and the command reports success. Failing by default + stops a deleted or mis-pathed reference from silently passing in CI. +- `--timeout-ms ` (default 30000, `-1` = never; also `TIMEOUT_MS`) bounds the + whole run. A watchdog reports the timeout and exits `1`, since a hung plugin + call can't be interrupted. +- If the reference exists, the rendered output is **compared** against it and + the command exits `0` (match) or `1` (mismatch), consistent with `validate`. + +### Relationship to `--config` (important) + +Do **not** confuse the acceptance-test config with the existing `--config` +flag. They are unrelated: + +| | `--config file.json` | `pluginval test file.json` | +|---|---|---| +| Purpose | A *settings layer* for a **validate** run | The full **acceptance-test definition** | +| Schema | `PluginvalSettings` (strictness, timeouts, sample-rate lists, …) | Plugin + input + reference + state + comparison | +| Merge behaviour | `merge_patch`ed into the settings layering | Loaded standalone | +| How supplied | `--config` option | Positional argument to the `test` verb | + +The acceptance-test definition is **never** routed through the +`PluginvalSettings` / `--config` layering. + +## 3. Integration with the CLI pipeline + +The CLI was rewritten (PRs #175 and #176) onto **CLI11** + an **nlohmann/json** +settings pipeline with a subcommand dispatcher. Acceptance testing plugs into +that dispatcher; it does **not** touch the validate settings-layering at all. + +Required edits (small and localised): + +| Location | Edit | +|---|---| +| `SettingsParser.h` — `enum class Command` | add `test` | +| `SettingsParser.cpp` — `dispatch()` | recognise the verb `test`, peel it, capture the positional config path(s) | +| `SettingsParser.cpp` — `isCommandLine()` | recognise `test` so CLI mode is triggered | +| `SettingsParser.cpp` — `getFooterText()` | add `test` to the `Commands:` help block | +| `CommandLine.cpp` — `performCommandLine()` | add a `Command::test` branch that runs the acceptance runner and async-quits | + +Because `test` is brand new there are **no deprecated-alias** concerns. + +### Process isolation + +For v1, acceptance tests run **in-process**: record and compare both need the +rendered buffer back in the calling process, so in-process is the natural +choice. The render runs on a background thread (`acceptance::TestRunner`) so the +message thread stays free to service the plugin (VST3 lifecycle calls, +async-only creation, work the plugin posts to the message thread); the plugin is +deleted on the message thread. Crash isolation can be layered on later by mirroring the validate +child-process handoff (`createChildProcessCommandLine` in `SettingsParser.cpp`, +which passes an authoritative base64-JSON blob via `--config-base64`); the +equivalent would be `test --config-base64 ` with the child writing the +reference / diff to disk and returning an exit code. + +## 4. Config schema + +The config is a plain, std-typed struct that follows the same **nlohmann/json +(de)serialisation pattern** as `PluginvalSettings.h`, plus a `toX()` boundary +conversion to JUCE types. **JSON keys are `snake_case`** (e.g. `sample_rate`). +Because the C++ members stay JUCE-style `camelCase`, the snake_case keys are +mapped explicitly via per-struct `to_json` / `from_json` (rather than the bare +`NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE_WITH_DEFAULT` macro, which would emit the +member names verbatim). Missing keys still fall back to the defaults. + +```jsonc +{ + "name": "myReverb-default", + "plugin": "/path/to/Plugin.vst3", // path or AU identifier string + "input": { "audio": "in.wav", "midi": "in.mid" }, // either / both / omitted -> silence + "reference": "refs/myReverb-default.wav", // optional; default derived from "name" + "state": { "file": "preset.state" }, // OR "parameters": { "Mix": 0.5, "3": 1.0 } + "sample_rate": 48000, + "block_size": 512, + "render_duration": 2.0, // seconds; omitted -> use input length + "comparison": { "sample": 1e-6 }, // omitted -> default 1/32768 (16-bit LSB) + "playhead": { "bpm": 120, "time_signature": { "numerator": 4, "denominator": 4 } }, + "automation": [ /* phase 2 */ ] +} +``` + +### Field reference + +| Field | Type | Default | Notes | +|---|---|---|---| +| `name` | string | reference's basename | Used to derive the default reference path and to label results. `name` or `reference` is required. | +| `plugin` | string | (required) | Absolute/relative path, or an AU identifier string. Relative paths resolve against the working directory. | +| `input.audio` | string | none | Path to an input audio file (WAV/AIFF/FLAC via `AudioFormatManager`). Used for effects. Its sample rate must equal `sample_rate`. File channel N feeds plugin input channel N; a mono file feeds channel 0 only (deliberately not duplicated). | +| `input.midi` | string | none | Path to a `.mid` file (`juce::MidiFile`). Used for instruments. Meta events are skipped. | +| `reference` | string | `/.wav` | The golden file. Absent -> record mode; present -> compare mode. | +| `state.file` | string | none | Binary blob from `getStateInformation`, applied via the VST3-safe helper. | +| `state.parameters` | object | none | `name-or-index -> normalised value`, applied in file order (parsed with `nlohmann::ordered_json`). Applied after `state.file` if both are given. Digits-only keys are indices; otherwise the format's parameter ID, then the display name, case-insensitively. | +| `sample_rate` | number | 44100 | Single value (not a list, unlike validate). Must be positive and finite. | +| `block_size` | number | 512 | Single value, 1 to 65536. | +| `render_duration` | number | input length | Seconds. `num_samples = round(duration * sample_rate)`. Input shorter than the duration is padded with silence; longer is truncated. | +| `comparison` | object | `{ "sample": 0.0000305 }` | Map of comparator name -> its sub-config. See §5. An empty object is rejected. The `sample` comparator fails on any NaN / inf difference. | +| `playhead` | object | none | Fixed transport for time-dependent plugins. `{ "bpm": , "time_signature": { "numerator": N, "denominator": D }, "start_ppq": }`. `time_signature` defaults to 4/4, `start_ppq` to 0. Omitted -> no playhead is set (the plugin sees `getPlayHead() == nullptr`). The tempo / time signature are constant; the position advances with the render. | +| `automation` | array | none | **Phase 2.** Parameter changes scheduled at sample positions. | + +### State precedence + +If both `state.file` and `state.parameters` are present, the binary state is +applied **first** (it represents the full plugin state), then the parameter map +is applied on top as overrides. + +### Multiplexing (future) + +The loader accepts **either a single object or a top-level array** of configs. +v1 may execute only the single / first entry; phase 2 iterates the array to +multiplex many test cases from one file. Designing the parser for arrays now +costs nothing and avoids a schema break later. Each array entry should carry a +`name` so its reference path and result are uniquely identifiable. + +## 5. Comparator abstraction (the core extension point) + +Comparison is **pluggable**. The config selects one or more methods; v1 ships +only `sample`, and additional methods register later with **zero** changes to +config parsing or the runner. + +```cpp +struct ComparisonResult +{ + bool passed = false; + double score = 0.0; // method-specific metric (e.g. max abs diff) + juce::String summary; // human-readable + nlohmann::json details; // structured, for the machine-readable report +}; + +struct Comparator +{ + virtual ~Comparator() = default; + virtual juce::String getName() const = 0; // "sample", "spectrum", ... + virtual ComparisonResult compare (const juce::AudioBuffer& reference, + const juce::AudioBuffer& output, + const nlohmann::json& config) = 0; // value under comparison[name] +}; +``` + +- A small **registry** maps method name -> factory. +- `config` is whatever sits under the comparator's key — a bare number + (`1e-6`) for `sample`, or an object for richer methods + (e.g. `"spectrum": { "tolerance": 0.05, "fftSize": 2048 }`). +- When multiple methods are listed, **all must pass** (logical AND); each is + reported separately. + +### `comparison` defaults + +When `comparison` is omitted, the default is: + +```json +"comparison": { "sample": 0.0000305 } // 1.0 / 32768.0, i.e. one 16-bit LSB +``` + +### Roadmap of comparators + +| Method | Phase | Description | +|---|---|---| +| `sample` | **v1** | Per-sample absolute-difference tolerance (`0` = bit-exact) plus a length check. | +| `peakrms` | future | Peak and RMS difference thresholds. | +| `spectrum` | future | FFT-magnitude comparison with tolerance; robust to small phase/SIMD differences. | +| `crosscorr` | future | Cross-correlation, tolerant to small latency offsets. | +| `fingerprint` | future | Acoustic fingerprint match; most robust for cross-platform/version. | + +The `spectrum` / `crosscorr` / `fingerprint` methods are what make the +**cross-platform-portable** reference goal practical. + +## 6. Reference artifact format + +A reference is **two files**: + +1. **`.wav`** — the rendered output as a **32-bit float WAV** (preserves + full precision; it is the source of truth for comparison). +2. **`.wav.json`** — a sidecar manifest: + +```jsonc +{ + "plugin": { "name": "...", "manufacturer": "...", "version": "...", + "format": "VST3", "uid": "..." }, + "render": { "sample_rate": 48000, "block_size": 512, "num_channels": 2, + "num_samples": 96000, "length_seconds": 2.0 }, + "pluginval_version": "2.0.0", + "config_hash": "…", // XXH3 of the config, paths excluded (see below) + "created_on": { "os": "macOS", "arch": "arm64", "date": "2026-…" } // informational only +} +``` + +- `created_on` is **informational only** — it is never used for matching, because + references are meant to be portable and shared. +- `config_hash` lets the runner detect a **stale** reference (one produced from a + different config than the one now being run) and warn. It is an XXH3-64 hash + (xxHash, header-only via CPM) of the config with the `plugin` and `reference` + paths removed and the `input.audio` / `input.midi` / `state.file` paths replaced + by XXH3 hashes of the files' contents. Paths are machine-specific, so hashing + them would make every checked-in reference look stale elsewhere; the plugin + binary is what's under test, so it is deliberately not part of the hash. +- The default reference path deliberately contains **no platform/arch**, since + references are intended to be portable and checked into a repo. + +### Diff output on failure + +When a comparison fails, the runner also writes a **diff WAV** (`output − reference`) +next to the result to aid debugging. + +## 7. Module layout + +``` +source/acceptance/ + TestConfig.h/.cpp // std-typed struct + NLOHMANN_DEFINE_TYPE..._WITH_DEFAULT + // + toRenderSpec() boundary conversion (mirrors PluginvalSettings.h) + AcceptanceTest.h/.cpp // resolve + load plugin, apply state, feed input, render -> buffer + ReferenceComparator.h/.cpp // Comparator interface + registry + SampleComparator; record-or-compare + TestReporter.h/.cpp // text + JSON result, exit code +``` + +Reused infrastructure: + +- Plugin loading / scanning: `AudioPluginFormatManager::createPluginInstance` + and `KnownPluginList` (as in `PluginTests.cpp`). +- VST3-safe lifecycle helpers in `TestUtilities.h`: + `callPrepareToPlayOnMessageThreadIfVST3`, + `callSetStateInformationOnMessageThreadIfVST3`, etc. (VST3 requires several + operations on the message thread). +- The render-loop shape from the `AudioProcessingTest` in + `source/tests/BasicTests.cpp` (`prepareToPlay` -> `processBlock` loop with + `AudioBuffer` + `MidiBuffer`). + +Build wiring: add the new `.cpp` files to the `SourceFiles` list in +`CMakeLists.txt`, and add acceptance-mode cases to `source/CommandLineTests.cpp` +following the existing test style. + +## 8. Dogfood test plugin (tone generator) + +To develop and self-test the acceptance feature we need a **device under test +that is fully deterministic** — something whose output is known exactly, doesn't +depend on a third-party binary, and is stable across runs and platforms. We +build a tiny in-repo **tone-generator plugin** for this purpose ("dogfooding" +the feature with our own plugin). + +### What it is + +A minimal JUCE plugin (built with `juce_add_plugin`) that synthesises a simple, +deterministic tone: + +- **Parameters** (so we can exercise state / parameter application): + - `waveform` — choice: `sine`, `square` (extendable to `saw`, `triangle`). + - `frequency` — Hz (e.g. 20–20000, default 440). + - `gain` — linear or dB (default −6 dB). +- **Determinism**: the oscillator **phase resets to 0 on `prepareToPlay`** and + the waveform is computed in closed form from the sample index, so the same + config always renders the identical buffer (ideal for the `sample` + comparator, including bit-exact). No randomness, no denormal-sensitive + feedback paths. +- **No audio input required**: it is a generator, so acceptance configs for it + use silence input (or omit `input`) and a fixed `render_duration`. +- Produces a VST3 on all platforms (and an AU on macOS) so the same plugin + exercises both formats. + +### Where it lives + +A new CMake target alongside the existing `tests/test_plugins/`, e.g. +`pluginval_tone_generator`, built on demand (guarded behind an option such as +`PLUGINVAL_BUILD_TEST_PLUGINS`, off for normal release builds). Suggested +layout: + +``` +tests/test_plugins/tone_generator/ + CMakeLists.txt // juce_add_plugin target + ToneGeneratorPlugin.h/.cpp +``` + +### How it self-tests the acceptance feature + +Check in a handful of acceptance configs plus their recorded reference WAVs and +wire them into CTest, so CI both validates the tone generator's stability and +exercises the full record/compare path: + +``` +tests/acceptance/ + sine-440.json.in // tone gen, waveform=sine, state.parameters + square-220.json.in // tone gen, waveform=square, state.parameters, bit-exact + square-state.json.in // tone gen, state.file blob + a gain parameter override + gain-half.json.in // gain effect, input.audio = a full-height sine, bit-exact + playhead-120.json.in // playhead probe, fixed transport (bpm 120, 4/4), bit-exact + inputs/sine-full.wav // checked-in input for gain-half (±1.0 sine) + refs/.wav (+ .wav.json) // checked-in references + sidecar manifests + refs/square-state.state // checked-in getStateInformation blob +``` + +The configs are checked-in **templates** (`*.json.in`): the plugin artefact +paths and the input/reference directories are substituted at CMake configure +time (the tests can't hardcode a build-tree plugin path). Each `pluginval test +` CTest case asserts exit code `0` against the checked-in reference. +Because the dogfood plugins are closed-form deterministic, these references are +stable enough to commit — a first smoke test of cross-platform portability for +the `sample` comparator and the baseline for future comparators (`spectrum`, +`crosscorr`, …). + +The five cases cover the distinct render paths: + +- **`sine-440` / `square-220`** — the `state.parameters` (name/index → normalised + value) path. `square-220` compares bit-exact (`"sample": 0`). +- **`square-state`** — the binary `state.file` (`setStateInformation`) path, + plus a `state.parameters` `gain` override on top, exercising the + state-then-parameters precedence of §4. The blob is captured once and checked + in alongside its reference WAV. +- **`gain-half`** — the **`input.audio`** effect path: a second dogfood plugin + (`tests/test_plugins/gain/`, a deterministic gain effect) gains a checked-in + full-height (±1.0) sine by 0.5 and is compared bit-exact (0.5 is exact in + float). It also omits `render_duration`, so the render length is derived from + the input file. +- **`playhead-120`** — the **`playhead`** path: a third dogfood plugin + (`tests/test_plugins/playhead_probe/`) writes the host transport into its + output (channel 0 = `ppqPosition`, channel 1 = tempo), so the recorded + reference is a direct check that the fixed transport reached the plugin. If the + playhead regressed, the probe would output silence and the compare would fail. + +## 9. Execution flow + +1. Parse `pluginval test ` into one or more `TestConfig`. +2. Resolve the plugin (path or ID); create the instance at the config's + `sample_rate` / `block_size`. +3. Apply `state.file` then `state.parameters`. +4. Load `input.audio` and/or `input.midi`; otherwise use silence. +5. If a `playhead` is configured, point a fixed-tempo transport at the plugin + (its position advances each block). +6. `prepareToPlay`, render `render_duration` worth of blocks, accumulating the + output into a single buffer. +7. **No reference exists** -> without `--record-missing`, fail before rendering; + with it, write the float WAV + manifest (record mode) and report + "reference created". +8. **Reference exists** -> run each configured comparator; report each verdict + and the overall pass/fail; on failure write a diff WAV; exit `0` / `1`. + +## 10. Phasing + +**Phase 1 (initial implementation)** + +- `pluginval test ` subcommand wired into `settings_parser`. +- Single-config (object) execution; parser already accepts arrays. +- Plugin load + state (file and/or parameter map) + file-based audio/MIDI input + (or silence). +- Fixed-duration render in-process. +- Record-or-compare with float-WAV + JSON-sidecar references. +- `sample` comparator only (default tolerance = one 16-bit LSB). +- Text + JSON result reporting; diff WAV on failure. +- Fixed `playhead` (tempo / time signature) for time-dependent plugins. +- **Dogfood test plugins** (§8) plus CTest self-tests that run the full + record/compare path against checked-in references. + +**Phase 2 and beyond** + +- Multiplexed execution of config arrays. +- Parameter `automation` timelines. +- Additional comparators: `peakrms`, `spectrum`, `crosscorr`, `fingerprint`. +- Synthesised inputs (`input.generator`: noise / sine, with seed). +- Optional child-process isolation mirroring the validate handoff. diff --git a/tests/acceptance/CMakeLists.txt b/tests/acceptance/CMakeLists.txt new file mode 100644 index 00000000..5d1f350c --- /dev/null +++ b/tests/acceptance/CMakeLists.txt @@ -0,0 +1,39 @@ +# Acceptance self-tests: drive the dogfood plugins through the full record/compare +# path against checked-in reference WAVs. +# +# The configs are checked-in templates (*.json.in). At configure time the dogfood +# plugin artefact paths, the input directory and the (source-tree) reference +# directory are substituted in. An artefact path may contain a generator +# expression (e.g. $ on multi-config generators), so we resolve it via +# file(GENERATE). + +get_target_property(tone_artefact pluginval_tone_generator_VST3 JUCE_PLUGIN_ARTEFACT_FILE) +get_target_property(gain_artefact pluginval_gain_VST3 JUCE_PLUGIN_ARTEFACT_FILE) +get_target_property(probe_artefact pluginval_playhead_probe_VST3 JUCE_PLUGIN_ARTEFACT_FILE) + +set(ACCEPTANCE_REF_DIR "${CMAKE_CURRENT_SOURCE_DIR}/refs") +set(ACCEPTANCE_INPUT_DIR "${CMAKE_CURRENT_SOURCE_DIR}/inputs") +set(TONE_GENERATOR_PLUGIN "${tone_artefact}") +set(GAIN_PLUGIN "${gain_artefact}") +set(PLAYHEAD_PROBE_PLUGIN "${probe_artefact}") + +set(acceptance_cases + sine-440 # state.parameters path (waveform/frequency/gain by name) + square-220 # state.parameters path, bit-exact square wave + square-state # state.file (setStateInformation) + parameter override precedence + gain-half # input.audio effect path: gain a full-height sine, length from input + playhead-120) # fixed-playhead path: probe writes the host transport into the output + +foreach(case ${acceptance_cases}) + # Step 1: @-substitute the reference dir (and the possibly-genex plugin path). + configure_file("${CMAKE_CURRENT_SOURCE_DIR}/${case}.json.in" + "${CMAKE_CURRENT_BINARY_DIR}/${case}.json.gen" @ONLY) + + # Step 2: resolve any generator expression left in the plugin path. The output + # path is per-config so multi-config generators don't collide on one file. + file(GENERATE OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/$/${case}.json" + INPUT "${CMAKE_CURRENT_BINARY_DIR}/${case}.json.gen") + + add_test(NAME "pluginval.acceptance.${case}" + COMMAND pluginval test "${CMAKE_CURRENT_BINARY_DIR}/$/${case}.json") +endforeach() diff --git a/tests/acceptance/gain-half.json.in b/tests/acceptance/gain-half.json.in new file mode 100644 index 00000000..1fa0970f --- /dev/null +++ b/tests/acceptance/gain-half.json.in @@ -0,0 +1,16 @@ +{ + "name": "gain-half", + "plugin": "@GAIN_PLUGIN@", + "input": { "audio": "@ACCEPTANCE_INPUT_DIR@/sine-full.wav" }, + "reference": "@ACCEPTANCE_REF_DIR@/gain-half.wav", + "state": { + "parameters": { + "gain": 0.5 + } + }, + "sample_rate": 48000, + "block_size": 512, + "comparison": { + "sample": 0.0 + } +} diff --git a/tests/acceptance/inputs/sine-full.wav b/tests/acceptance/inputs/sine-full.wav new file mode 100644 index 00000000..6cf57322 Binary files /dev/null and b/tests/acceptance/inputs/sine-full.wav differ diff --git a/tests/acceptance/playhead-120.json.in b/tests/acceptance/playhead-120.json.in new file mode 100644 index 00000000..d8aacd6f --- /dev/null +++ b/tests/acceptance/playhead-120.json.in @@ -0,0 +1,15 @@ +{ + "name": "playhead-120", + "plugin": "@PLAYHEAD_PROBE_PLUGIN@", + "reference": "@ACCEPTANCE_REF_DIR@/playhead-120.wav", + "sample_rate": 48000, + "block_size": 512, + "render_duration": 0.25, + "playhead": { + "bpm": 120, + "time_signature": { "numerator": 4, "denominator": 4 } + }, + "comparison": { + "sample": 0.0 + } +} diff --git a/tests/acceptance/refs/gain-half.wav b/tests/acceptance/refs/gain-half.wav new file mode 100644 index 00000000..8d419d31 Binary files /dev/null and b/tests/acceptance/refs/gain-half.wav differ diff --git a/tests/acceptance/refs/gain-half.wav.json b/tests/acceptance/refs/gain-half.wav.json new file mode 100644 index 00000000..ad69ab70 --- /dev/null +++ b/tests/acceptance/refs/gain-half.wav.json @@ -0,0 +1,23 @@ +{ + "config_hash": "19abbba838eff61e", + "created_on": { + "arch": "arm64", + "date": "2026-06-16T14:22:49.718+01:00", + "os": "Mac OSX 26.2" + }, + "plugin": { + "format": "VST3", + "manufacturer": "Tracktion", + "name": "pluginval Gain", + "uid": "VST3-pluginval Gain-327cfb94-d8aa82c7", + "version": "1.0.4" + }, + "pluginval_version": "1.0.4", + "render": { + "block_size": 512, + "length_seconds": 0.25, + "num_channels": 2, + "num_samples": 12000, + "sample_rate": 48000.0 + } +} \ No newline at end of file diff --git a/tests/acceptance/refs/playhead-120.wav b/tests/acceptance/refs/playhead-120.wav new file mode 100644 index 00000000..a98a8764 Binary files /dev/null and b/tests/acceptance/refs/playhead-120.wav differ diff --git a/tests/acceptance/refs/playhead-120.wav.json b/tests/acceptance/refs/playhead-120.wav.json new file mode 100644 index 00000000..7a25f3b1 --- /dev/null +++ b/tests/acceptance/refs/playhead-120.wav.json @@ -0,0 +1,23 @@ +{ + "config_hash": "2756114d8a1871ff", + "created_on": { + "arch": "arm64", + "date": "2026-06-16T15:51:01.553+01:00", + "os": "Mac OSX 26.2" + }, + "plugin": { + "format": "VST3", + "manufacturer": "Tracktion", + "name": "pluginval Playhead Probe", + "uid": "VST3-pluginval Playhead Probe-f5ce8b6f-d8b37cc7", + "version": "1.0.4" + }, + "pluginval_version": "1.0.4", + "render": { + "block_size": 512, + "length_seconds": 0.25, + "num_channels": 2, + "num_samples": 12000, + "sample_rate": 48000.0 + } +} \ No newline at end of file diff --git a/tests/acceptance/refs/sine-440.wav b/tests/acceptance/refs/sine-440.wav new file mode 100644 index 00000000..8d419d31 Binary files /dev/null and b/tests/acceptance/refs/sine-440.wav differ diff --git a/tests/acceptance/refs/sine-440.wav.json b/tests/acceptance/refs/sine-440.wav.json new file mode 100644 index 00000000..07ab0488 --- /dev/null +++ b/tests/acceptance/refs/sine-440.wav.json @@ -0,0 +1,23 @@ +{ + "config_hash": "cfada3428594ae2a", + "created_on": { + "arch": "arm64", + "date": "2026-06-16T13:10:20.117+01:00", + "os": "Mac OSX 26.2" + }, + "plugin": { + "format": "VST3", + "manufacturer": "Tracktion", + "name": "pluginval Tone Generator", + "uid": "VST3-pluginval Tone Generator-d86456ee-d8b77bc7", + "version": "1.0.4" + }, + "pluginval_version": "1.0.4", + "render": { + "block_size": 512, + "length_seconds": 0.25, + "num_channels": 2, + "num_samples": 12000, + "sample_rate": 48000.0 + } +} \ No newline at end of file diff --git a/tests/acceptance/refs/square-220.wav b/tests/acceptance/refs/square-220.wav new file mode 100644 index 00000000..4032e1df Binary files /dev/null and b/tests/acceptance/refs/square-220.wav differ diff --git a/tests/acceptance/refs/square-220.wav.json b/tests/acceptance/refs/square-220.wav.json new file mode 100644 index 00000000..edc05288 --- /dev/null +++ b/tests/acceptance/refs/square-220.wav.json @@ -0,0 +1,23 @@ +{ + "config_hash": "92e8e5abbb73ec76", + "created_on": { + "arch": "arm64", + "date": "2026-06-16T13:10:20.377+01:00", + "os": "Mac OSX 26.2" + }, + "plugin": { + "format": "VST3", + "manufacturer": "Tracktion", + "name": "pluginval Tone Generator", + "uid": "VST3-pluginval Tone Generator-d86456ee-d8b77bc7", + "version": "1.0.4" + }, + "pluginval_version": "1.0.4", + "render": { + "block_size": 512, + "length_seconds": 0.25, + "num_channels": 2, + "num_samples": 12000, + "sample_rate": 48000.0 + } +} \ No newline at end of file diff --git a/tests/acceptance/refs/square-state.state b/tests/acceptance/refs/square-state.state new file mode 100644 index 00000000..5a668c94 Binary files /dev/null and b/tests/acceptance/refs/square-state.state differ diff --git a/tests/acceptance/refs/square-state.wav b/tests/acceptance/refs/square-state.wav new file mode 100644 index 00000000..9dcab4c2 Binary files /dev/null and b/tests/acceptance/refs/square-state.wav differ diff --git a/tests/acceptance/refs/square-state.wav.json b/tests/acceptance/refs/square-state.wav.json new file mode 100644 index 00000000..38823684 --- /dev/null +++ b/tests/acceptance/refs/square-state.wav.json @@ -0,0 +1,23 @@ +{ + "config_hash": "2ccce4612b4db9fc", + "created_on": { + "arch": "arm64", + "date": "2026-06-16T13:10:20.606+01:00", + "os": "Mac OSX 26.2" + }, + "plugin": { + "format": "VST3", + "manufacturer": "Tracktion", + "name": "pluginval Tone Generator", + "uid": "VST3-pluginval Tone Generator-d86456ee-d8b77bc7", + "version": "1.0.4" + }, + "pluginval_version": "1.0.4", + "render": { + "block_size": 512, + "length_seconds": 0.25, + "num_channels": 2, + "num_samples": 12000, + "sample_rate": 48000.0 + } +} \ No newline at end of file diff --git a/tests/acceptance/sine-440.json.in b/tests/acceptance/sine-440.json.in new file mode 100644 index 00000000..35650ef6 --- /dev/null +++ b/tests/acceptance/sine-440.json.in @@ -0,0 +1,15 @@ +{ + "name": "sine-440", + "plugin": "@TONE_GENERATOR_PLUGIN@", + "reference": "@ACCEPTANCE_REF_DIR@/sine-440.wav", + "state": { + "parameters": { + "waveform": 0.0, + "frequency": 0.021021021021021, + "gain": 0.5 + } + }, + "sample_rate": 48000, + "block_size": 512, + "render_duration": 0.25 +} diff --git a/tests/acceptance/square-220.json.in b/tests/acceptance/square-220.json.in new file mode 100644 index 00000000..d0706873 --- /dev/null +++ b/tests/acceptance/square-220.json.in @@ -0,0 +1,18 @@ +{ + "name": "square-220", + "plugin": "@TONE_GENERATOR_PLUGIN@", + "reference": "@ACCEPTANCE_REF_DIR@/square-220.wav", + "state": { + "parameters": { + "waveform": 1.0, + "frequency": 0.010010010010010, + "gain": 0.5 + } + }, + "sample_rate": 48000, + "block_size": 512, + "render_duration": 0.25, + "comparison": { + "sample": 0.0 + } +} diff --git a/tests/acceptance/square-state.json.in b/tests/acceptance/square-state.json.in new file mode 100644 index 00000000..c30efebd --- /dev/null +++ b/tests/acceptance/square-state.json.in @@ -0,0 +1,17 @@ +{ + "name": "square-state", + "plugin": "@TONE_GENERATOR_PLUGIN@", + "reference": "@ACCEPTANCE_REF_DIR@/square-state.wav", + "state": { + "file": "@ACCEPTANCE_REF_DIR@/square-state.state", + "parameters": { + "gain": 0.25 + } + }, + "sample_rate": 48000, + "block_size": 512, + "render_duration": 0.25, + "comparison": { + "sample": 0.0 + } +} diff --git a/tests/pluginval_as_dependency/CMakeLists.txt b/tests/pluginval_as_dependency/CMakeLists.txt index 0ea3743e..01d22b77 100644 --- a/tests/pluginval_as_dependency/CMakeLists.txt +++ b/tests/pluginval_as_dependency/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 3.22) project(PluginvalAsDependency VERSION 1.0.0) -set(JUCE_VERSION "8.0.3" CACHE STRING "JUCE version to test against") +set(JUCE_VERSION "9.0.3" CACHE STRING "JUCE version to test against") include(${CMAKE_CURRENT_LIST_DIR}/../../cmake/CPM.cmake) diff --git a/tests/test_plugins/gain/CMakeLists.txt b/tests/test_plugins/gain/CMakeLists.txt new file mode 100644 index 00000000..ec2f43d3 --- /dev/null +++ b/tests/test_plugins/gain/CMakeLists.txt @@ -0,0 +1,30 @@ +# A minimal, fully deterministic gain effect used to dogfood the acceptance +# feature's audio-input path. Built only when PLUGINVAL_BUILD_TEST_PLUGINS is ON. + +juce_add_plugin(pluginval_gain + PRODUCT_NAME "pluginval Gain" + COMPANY_NAME Tracktion + BUNDLE_ID com.Tracktion.pluginvalGain + PLUGIN_MANUFACTURER_CODE Trkt + PLUGIN_CODE Pgn1 + FORMATS VST3 $<$:AU> + IS_SYNTH FALSE + NEEDS_MIDI_INPUT FALSE + COPY_PLUGIN_AFTER_BUILD FALSE) + +target_sources(pluginval_gain PRIVATE + GainPlugin.cpp + GainPlugin.h) + +target_compile_features(pluginval_gain PRIVATE cxx_std_20) + +target_compile_definitions(pluginval_gain PRIVATE + JUCE_WEB_BROWSER=0 + JUCE_USE_CURL=0 + JUCE_VST3_CAN_REPLACE_VST2=0) + +target_link_libraries(pluginval_gain PRIVATE + juce::juce_audio_utils + juce::juce_audio_plugin_client + juce::juce_recommended_config_flags + juce::juce_recommended_warning_flags) diff --git a/tests/test_plugins/gain/GainPlugin.cpp b/tests/test_plugins/gain/GainPlugin.cpp new file mode 100644 index 00000000..fb3af1de --- /dev/null +++ b/tests/test_plugins/gain/GainPlugin.cpp @@ -0,0 +1,56 @@ +/*============================================================================== + + Copyright 2018 by Tracktion Corporation. + For more information visit www.tracktion.com + + You may also use this code under the terms of the GPL v3 (see + www.gnu.org/licenses). + + pluginval IS PROVIDED "AS IS" WITHOUT ANY WARRANTY, AND ALL WARRANTIES, WHETHER + EXPRESSED OR IMPLIED, INCLUDING MERCHANTABILITY AND FITNESS FOR PURPOSE, ARE + DISCLAIMED. + + ==============================================================================*/ + +#include "GainPlugin.h" + +//============================================================================== +GainProcessor::GainProcessor() + : juce::AudioProcessor (BusesProperties().withInput ("Input", juce::AudioChannelSet::stereo(), true) + .withOutput ("Output", juce::AudioChannelSet::stereo(), true)) +{ + // Default gain of 1.0 passes the input through unchanged. + addParameter (gain = new juce::AudioParameterFloat ("gain", "Gain", + juce::NormalisableRange (0.0f, 1.0f), 1.0f)); +} + +//============================================================================== +void GainProcessor::processBlock (juce::AudioBuffer& buffer, juce::MidiBuffer&) +{ + const float g = gain->get(); + + for (int c = 0; c < buffer.getNumChannels(); ++c) + buffer.applyGain (c, 0, buffer.getNumSamples(), g); +} + +//============================================================================== +void GainProcessor::getStateInformation (juce::MemoryBlock& destData) +{ + // getValue() is a private override on the concrete type, so go via the base. + juce::MemoryOutputStream mos (destData, false); + mos.writeFloat (static_cast (gain)->getValue()); +} + +void GainProcessor::setStateInformation (const void* data, int sizeInBytes) +{ + juce::MemoryInputStream mis (data, (size_t) sizeInBytes, false); + + if (mis.getNumBytesRemaining() >= (int) sizeof (float)) + gain->setValueNotifyingHost (mis.readFloat()); +} + +//============================================================================== +juce::AudioProcessor* JUCE_CALLTYPE createPluginFilter() +{ + return new GainProcessor(); +} diff --git a/tests/test_plugins/gain/GainPlugin.h b/tests/test_plugins/gain/GainPlugin.h new file mode 100644 index 00000000..27446fac --- /dev/null +++ b/tests/test_plugins/gain/GainPlugin.h @@ -0,0 +1,68 @@ +/*============================================================================== + + Copyright 2018 by Tracktion Corporation. + For more information visit www.tracktion.com + + You may also use this code under the terms of the GPL v3 (see + www.gnu.org/licenses). + + pluginval IS PROVIDED "AS IS" WITHOUT ANY WARRANTY, AND ALL WARRANTIES, WHETHER + EXPRESSED OR IMPLIED, INCLUDING MERCHANTABILITY AND FITNESS FOR PURPOSE, ARE + DISCLAIMED. + + ==============================================================================*/ + +#pragma once + +#include + +//============================================================================== +/** + A minimal, fully deterministic gain effect used to dogfood the acceptance + feature's audio-input path. + + Unlike the tone generator (a pure generator), this is an effect: it + multiplies its input by a single linear gain parameter. Feeding it a known + input file and comparing the gained output exercises the input.audio render + path. Multiplying by an exact-in-float gain (e.g. 0.5) keeps it bit-exact. +*/ +class GainProcessor : public juce::AudioProcessor +{ +public: + //============================================================================== + GainProcessor(); + ~GainProcessor() override = default; + + //============================================================================== + void prepareToPlay (double, int) override {} + void releaseResources() override {} + void processBlock (juce::AudioBuffer&, juce::MidiBuffer&) override; + + //============================================================================== + juce::AudioProcessorEditor* createEditor() override { return nullptr; } + bool hasEditor() const override { return false; } + + //============================================================================== + const juce::String getName() const override { return "pluginval Gain"; } + bool acceptsMidi() const override { return false; } + bool producesMidi() const override { return false; } + bool isMidiEffect() const override { return false; } + double getTailLengthSeconds() const override { return 0.0; } + + //============================================================================== + int getNumPrograms() override { return 1; } + int getCurrentProgram() override { return 0; } + void setCurrentProgram (int) override {} + const juce::String getProgramName (int) override { return "Default"; } + void changeProgramName (int, const juce::String&) override {} + + //============================================================================== + void getStateInformation (juce::MemoryBlock&) override; + void setStateInformation (const void*, int) override; + +private: + //============================================================================== + juce::AudioParameterFloat* gain = nullptr; + + JUCE_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR (GainProcessor) +}; diff --git a/tests/test_plugins/playhead_probe/CMakeLists.txt b/tests/test_plugins/playhead_probe/CMakeLists.txt new file mode 100644 index 00000000..c8c3660d --- /dev/null +++ b/tests/test_plugins/playhead_probe/CMakeLists.txt @@ -0,0 +1,33 @@ +# A minimal, deterministic plugin that writes the host transport into its output, +# used to dogfood the acceptance feature's fixed-playhead support. Built only when +# PLUGINVAL_BUILD_TEST_PLUGINS is ON. + +juce_add_plugin(pluginval_playhead_probe + PRODUCT_NAME "pluginval Playhead Probe" + COMPANY_NAME Tracktion + BUNDLE_ID com.Tracktion.pluginvalPlayheadProbe + PLUGIN_MANUFACTURER_CODE Trkt + PLUGIN_CODE Pph1 + FORMATS VST3 $<$:AU> + IS_SYNTH FALSE + NEEDS_MIDI_INPUT FALSE + VST3_CATEGORIES Generator + AU_MAIN_TYPE kAudioUnitType_Generator + COPY_PLUGIN_AFTER_BUILD FALSE) + +target_sources(pluginval_playhead_probe PRIVATE + PlayheadProbePlugin.cpp + PlayheadProbePlugin.h) + +target_compile_features(pluginval_playhead_probe PRIVATE cxx_std_20) + +target_compile_definitions(pluginval_playhead_probe PRIVATE + JUCE_WEB_BROWSER=0 + JUCE_USE_CURL=0 + JUCE_VST3_CAN_REPLACE_VST2=0) + +target_link_libraries(pluginval_playhead_probe PRIVATE + juce::juce_audio_utils + juce::juce_audio_plugin_client + juce::juce_recommended_config_flags + juce::juce_recommended_warning_flags) diff --git a/tests/test_plugins/playhead_probe/PlayheadProbePlugin.cpp b/tests/test_plugins/playhead_probe/PlayheadProbePlugin.cpp new file mode 100644 index 00000000..b94eb2b9 --- /dev/null +++ b/tests/test_plugins/playhead_probe/PlayheadProbePlugin.cpp @@ -0,0 +1,55 @@ +/*============================================================================== + + Copyright 2018 by Tracktion Corporation. + For more information visit www.tracktion.com + + You may also use this code under the terms of the GPL v3 (see + www.gnu.org/licenses). + + pluginval IS PROVIDED "AS IS" WITHOUT ANY WARRANTY, AND ALL WARRANTIES, WHETHER + EXPRESSED OR IMPLIED, INCLUDING MERCHANTABILITY AND FITNESS FOR PURPOSE, ARE + DISCLAIMED. + + ==============================================================================*/ + +#include "PlayheadProbePlugin.h" + +//============================================================================== +void PlayheadProbeProcessor::processBlock (juce::AudioBuffer& buffer, juce::MidiBuffer&) +{ + buffer.clear(); + + auto* playHead = getPlayHead(); + + if (playHead == nullptr) + return; + + const auto position = playHead->getPosition(); + + if (! position.hasValue()) + return; + + double ppq = 0.0, bpm = 0.0; + + if (const auto v = position->getPpqPosition()) + ppq = *v; + + if (const auto v = position->getBpm()) + bpm = *v; + + const int numSamples = buffer.getNumSamples(); + + // Channel 0: the block's ppqPosition (a per-block staircase that advances with + // the transport). Channel 1: the tempo, scaled into a sane range. + if (buffer.getNumChannels() > 0) + juce::FloatVectorOperations::fill (buffer.getWritePointer (0), (float) ppq, numSamples); + + if (buffer.getNumChannels() > 1) + juce::FloatVectorOperations::fill (buffer.getWritePointer (1), (float) (bpm / 1000.0), numSamples); +} + +//============================================================================== +juce::AudioProcessor* JUCE_CALLTYPE createPluginFilter() +{ + return new PlayheadProbeProcessor(); +} diff --git a/tests/test_plugins/playhead_probe/PlayheadProbePlugin.h b/tests/test_plugins/playhead_probe/PlayheadProbePlugin.h new file mode 100644 index 00000000..d3f03781 --- /dev/null +++ b/tests/test_plugins/playhead_probe/PlayheadProbePlugin.h @@ -0,0 +1,69 @@ +/*============================================================================== + + Copyright 2018 by Tracktion Corporation. + For more information visit www.tracktion.com + + You may also use this code under the terms of the GPL v3 (see + www.gnu.org/licenses). + + pluginval IS PROVIDED "AS IS" WITHOUT ANY WARRANTY, AND ALL WARRANTIES, WHETHER + EXPRESSED OR IMPLIED, INCLUDING MERCHANTABILITY AND FITNESS FOR PURPOSE, ARE + DISCLAIMED. + + ==============================================================================*/ + +#pragma once + +#include + +//============================================================================== +/** + A minimal, deterministic plugin that writes the host transport into its + output so the acceptance feature's fixed-playhead support can be verified. + + Each block it reads getPlayHead()->getPosition() and writes the block's + ppqPosition into channel 0 and the tempo (scaled) into channel 1. With no + playhead (or no position) it outputs silence - so a recorded reference is a + direct check that the transport actually reached the plugin. +*/ +class PlayheadProbeProcessor : public juce::AudioProcessor +{ +public: + //============================================================================== + PlayheadProbeProcessor() + : juce::AudioProcessor (BusesProperties().withOutput ("Output", juce::AudioChannelSet::stereo(), true)) + { + } + + ~PlayheadProbeProcessor() override = default; + + //============================================================================== + void prepareToPlay (double, int) override {} + void releaseResources() override {} + void processBlock (juce::AudioBuffer&, juce::MidiBuffer&) override; + + //============================================================================== + juce::AudioProcessorEditor* createEditor() override { return nullptr; } + bool hasEditor() const override { return false; } + + //============================================================================== + const juce::String getName() const override { return "pluginval Playhead Probe"; } + bool acceptsMidi() const override { return false; } + bool producesMidi() const override { return false; } + bool isMidiEffect() const override { return false; } + double getTailLengthSeconds() const override { return 0.0; } + + //============================================================================== + int getNumPrograms() override { return 1; } + int getCurrentProgram() override { return 0; } + void setCurrentProgram (int) override {} + const juce::String getProgramName (int) override { return "Default"; } + void changeProgramName (int, const juce::String&) override {} + + //============================================================================== + void getStateInformation (juce::MemoryBlock&) override {} + void setStateInformation (const void*, int) override {} + +private: + JUCE_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR (PlayheadProbeProcessor) +}; diff --git a/tests/test_plugins/tone_generator/CMakeLists.txt b/tests/test_plugins/tone_generator/CMakeLists.txt new file mode 100644 index 00000000..16a1e7c8 --- /dev/null +++ b/tests/test_plugins/tone_generator/CMakeLists.txt @@ -0,0 +1,32 @@ +# A minimal, fully deterministic tone-generator plugin used to dogfood the +# acceptance-testing feature. Built only when PLUGINVAL_BUILD_TEST_PLUGINS is ON. + +juce_add_plugin(pluginval_tone_generator + PRODUCT_NAME "pluginval Tone Generator" + COMPANY_NAME Tracktion + BUNDLE_ID com.Tracktion.pluginvalToneGenerator + PLUGIN_MANUFACTURER_CODE Trkt + PLUGIN_CODE Ptg1 + FORMATS VST3 $<$:AU> + IS_SYNTH FALSE + NEEDS_MIDI_INPUT FALSE + VST3_CATEGORIES Generator + AU_MAIN_TYPE kAudioUnitType_Generator + COPY_PLUGIN_AFTER_BUILD FALSE) + +target_sources(pluginval_tone_generator PRIVATE + ToneGeneratorPlugin.cpp + ToneGeneratorPlugin.h) + +target_compile_features(pluginval_tone_generator PRIVATE cxx_std_20) + +target_compile_definitions(pluginval_tone_generator PRIVATE + JUCE_WEB_BROWSER=0 + JUCE_USE_CURL=0 + JUCE_VST3_CAN_REPLACE_VST2=0) + +target_link_libraries(pluginval_tone_generator PRIVATE + juce::juce_audio_utils + juce::juce_audio_plugin_client + juce::juce_recommended_config_flags + juce::juce_recommended_warning_flags) diff --git a/tests/test_plugins/tone_generator/ToneGeneratorPlugin.cpp b/tests/test_plugins/tone_generator/ToneGeneratorPlugin.cpp new file mode 100644 index 00000000..ecc0565e --- /dev/null +++ b/tests/test_plugins/tone_generator/ToneGeneratorPlugin.cpp @@ -0,0 +1,114 @@ +/*============================================================================== + + Copyright 2018 by Tracktion Corporation. + For more information visit www.tracktion.com + + You may also use this code under the terms of the GPL v3 (see + www.gnu.org/licenses). + + pluginval IS PROVIDED "AS IS" WITHOUT ANY WARRANTY, AND ALL WARRANTIES, WHETHER + EXPRESSED OR IMPLIED, INCLUDING MERCHANTABILITY AND FITNESS FOR PURPOSE, ARE + DISCLAIMED. + + ==============================================================================*/ + +#include "ToneGeneratorPlugin.h" + +namespace +{ + constexpr double twoPi = 2.0 * 3.14159265358979323846; +} + +//============================================================================== +ToneGeneratorProcessor::ToneGeneratorProcessor() + : juce::AudioProcessor (BusesProperties().withOutput ("Output", juce::AudioChannelSet::stereo(), true)) +{ + // Parameter indices are stable (0, 1, 2) so acceptance configs can address + // them by index as well as by name. + addParameter (waveform = new juce::AudioParameterChoice ("waveform", "Waveform", + juce::StringArray { "sine", "square" }, 0)); + addParameter (frequency = new juce::AudioParameterFloat ("frequency", "Frequency", + juce::NormalisableRange (20.0f, 20000.0f), 440.0f)); + addParameter (gain = new juce::AudioParameterFloat ("gain", "Gain", + juce::NormalisableRange (0.0f, 1.0f), 0.5f)); +} + +//============================================================================== +void ToneGeneratorProcessor::prepareToPlay (double sampleRate, int) +{ + currentSampleRate = sampleRate; + sampleIndex = 0; // phase resets so renders are reproducible +} + +void ToneGeneratorProcessor::processBlock (juce::AudioBuffer& buffer, juce::MidiBuffer&) +{ + buffer.clear(); + + const auto numSamples = buffer.getNumSamples(); + const auto numChannels = buffer.getNumChannels(); + const bool isSquare = waveform->getIndex() == (int) Waveform::square; + const double freq = (double) frequency->get(); + const float g = gain->get(); + + if (numChannels > 0) + { + auto* channel0 = buffer.getWritePointer (0); + + for (int s = 0; s < numSamples; ++s) + { + const double phase = twoPi * freq * (double) (sampleIndex + s) / currentSampleRate; + + float value; + if (isSquare) + { + // Closed-form square: +1 for the first half of each cycle, -1 for the second. + const double fractional = phase / twoPi - std::floor (phase / twoPi); + value = fractional < 0.5 ? 1.0f : -1.0f; + } + else + { + value = (float) std::sin (phase); + } + + channel0[s] = value * g; + } + + // The tone is mono; copy it to every other output channel. + for (int c = 1; c < numChannels; ++c) + buffer.copyFrom (c, 0, buffer, 0, 0, numSamples); + } + + sampleIndex += numSamples; +} + +//============================================================================== +void ToneGeneratorProcessor::getStateInformation (juce::MemoryBlock& destData) +{ + // A tiny, explicit binary state: the three parameters' normalised values. + // This is the blob that a state.file acceptance test restores. getValue() is + // a private override on the concrete parameter types, so go via the base. + const auto normalised = [] (juce::AudioProcessorParameter* p) { return p->getValue(); }; + + juce::MemoryOutputStream mos (destData, false); + mos.writeFloat (normalised (waveform)); + mos.writeFloat (normalised (frequency)); + mos.writeFloat (normalised (gain)); +} + +void ToneGeneratorProcessor::setStateInformation (const void* data, int sizeInBytes) +{ + juce::MemoryInputStream mis (data, (size_t) sizeInBytes, false); + + if (mis.getNumBytesRemaining() >= (int) (3 * sizeof (float))) + { + waveform->setValueNotifyingHost (mis.readFloat()); + frequency->setValueNotifyingHost (mis.readFloat()); + gain->setValueNotifyingHost (mis.readFloat()); + } +} + +//============================================================================== +juce::AudioProcessor* JUCE_CALLTYPE createPluginFilter() +{ + return new ToneGeneratorProcessor(); +} diff --git a/tests/test_plugins/tone_generator/ToneGeneratorPlugin.h b/tests/test_plugins/tone_generator/ToneGeneratorPlugin.h new file mode 100644 index 00000000..04a6033d --- /dev/null +++ b/tests/test_plugins/tone_generator/ToneGeneratorPlugin.h @@ -0,0 +1,76 @@ +/*============================================================================== + + Copyright 2018 by Tracktion Corporation. + For more information visit www.tracktion.com + + You may also use this code under the terms of the GPL v3 (see + www.gnu.org/licenses). + + pluginval IS PROVIDED "AS IS" WITHOUT ANY WARRANTY, AND ALL WARRANTIES, WHETHER + EXPRESSED OR IMPLIED, INCLUDING MERCHANTABILITY AND FITNESS FOR PURPOSE, ARE + DISCLAIMED. + + ==============================================================================*/ + +#pragma once + +#include + +//============================================================================== +/** + A minimal, fully deterministic tone-generator plugin used to dogfood the + acceptance-testing feature. + + The output is computed in closed form from a sample index that resets to 0 + on prepareToPlay(), so the same configuration always renders the identical + buffer (ideal for the bit-exact / sample comparator). There is no randomness + and no denormal-sensitive feedback path. Audio input and MIDI are ignored - + it is a pure generator. +*/ +class ToneGeneratorProcessor : public juce::AudioProcessor +{ +public: + //============================================================================== + enum class Waveform { sine = 0, square = 1 }; + + ToneGeneratorProcessor(); + ~ToneGeneratorProcessor() override = default; + + //============================================================================== + void prepareToPlay (double sampleRate, int maximumExpectedSamplesPerBlock) override; + void releaseResources() override {} + void processBlock (juce::AudioBuffer&, juce::MidiBuffer&) override; + + //============================================================================== + juce::AudioProcessorEditor* createEditor() override { return nullptr; } + bool hasEditor() const override { return false; } + + //============================================================================== + const juce::String getName() const override { return "pluginval Tone Generator"; } + bool acceptsMidi() const override { return false; } + bool producesMidi() const override { return false; } + bool isMidiEffect() const override { return false; } + double getTailLengthSeconds() const override { return 0.0; } + + //============================================================================== + int getNumPrograms() override { return 1; } + int getCurrentProgram() override { return 0; } + void setCurrentProgram (int) override {} + const juce::String getProgramName (int) override { return "Default"; } + void changeProgramName (int, const juce::String&) override {} + + //============================================================================== + void getStateInformation (juce::MemoryBlock&) override; + void setStateInformation (const void*, int) override; + +private: + //============================================================================== + juce::AudioParameterChoice* waveform = nullptr; + juce::AudioParameterFloat* frequency = nullptr; + juce::AudioParameterFloat* gain = nullptr; + + double currentSampleRate = 44100.0; + juce::int64 sampleIndex = 0; // resets to 0 on prepareToPlay for determinism + + JUCE_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR (ToneGeneratorProcessor) +}; diff --git a/tests/windows_tests.bat b/tests/windows_tests.bat index 327d4766..e8ac04d7 100755 --- a/tests/windows_tests.bat +++ b/tests/windows_tests.bat @@ -64,11 +64,11 @@ exit /B %ERRORLEVEL% "%MSBUILD_EXE%" %PLUGIN_NAME%.sln /p:VisualStudioVersion=15.0 /m /t:Build /p:Configuration=Release /p:Platform=x64 /p:PreferredToolArchitecture=x64 /p:TreatWarningsAsErrors=true :: Test in process - call "%PLUGINVAL_EXE%" --validate-in-process --strictness-level 5 --validate %PLUGIN_VST3% + call "%PLUGINVAL_EXE%" validate --strictness-level 5 %PLUGIN_VST3% if %ERRORLEVEL% NEQ 0 exit 1 :: Test out of process - :: call "%PLUGINVAL_EXE%" --strictness-level 5 --validate %PLUGIN_VST3% + :: call "%PLUGINVAL_EXE%" validate --strictness-level 5 %PLUGIN_VST3% :: if %ERRORLEVEL% NEQ 0 exit 1 exit /B 0