From c187f16e6e6c56672cc301b26e898395fadad7eb Mon Sep 17 00:00:00 2001 From: Stuart Cameron Date: Sun, 25 Oct 2026 11:54:12 +1100 Subject: [PATCH 01/15] =?UTF-8?q?feat:=20anime=20filters=20=E2=80=94=20BM3?= =?UTF-8?q?D/KNLMeans,=20AI=20upscaling,=20Descale?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Noise Reduction: BM3D (16-bit, per-plane GRAY) and KNLMeans (own device probe: "gpu" then "auto"), advanced-only. BM3D r12 now ships on every platform (was macOS arm64 only). - Crop & Resize: Real-ESRGAN animevideo 2x/4x and Real-CUGAN 2x via vs-mlrt's VSNCNN (ncnn/Vulkan), tiled above SD. rife.rs generalised to gpu_plugin.rs; --probe-upscale. - Descale pass (after SpotLess): descale r12 on luma, NNEDI3 re-upscale, credit protection mask; --detect-native finds the native resolution and kernel. - CI: lavapipe on Linux/Windows (provision-vulkan.sh); with VAPOURBOX_EXPECT_VULKAN=1 the GPU tests fail instead of skipping. Co-Authored-By: Claude Opus 5.5 --- .github/scripts/provision-vulkan.sh | 64 ++ .github/workflows/ci-test.yml | 16 + .github/workflows/nightly.yml | 16 + CLAUDE.md | 184 ++++- README.md | 38 +- Scripts/deps-expected-plugins.json | 75 +- Scripts/download-deps-linux.sh | 186 ++++- Scripts/download-deps-macos.sh | 298 ++++++- Scripts/download-deps-windows.ps1 | 88 +- app/assets/filters/core/crop_resize.json | 51 +- app/assets/filters/core/descale.json | 265 ++++++ app/assets/filters/core/noise_reduction.json | 240 +++++- app/assets/filters/manifest.json | 3 +- app/lib/models/crop_resize_parameters.dart | 12 + app/lib/models/descale_parameters.dart | 103 +++ app/lib/models/gpu_plugin_status.dart | 97 +++ app/lib/models/native_res_report.dart | 78 ++ .../models/noise_reduction_parameters.dart | 70 +- app/lib/models/parameter_converter.dart | 79 +- app/lib/models/pass_relevance.dart | 3 + app/lib/models/processing_pipeline.dart | 24 + app/lib/models/rife_status.dart | 62 -- app/lib/services/worker_manager.dart | 89 +- app/lib/viewmodels/main_viewmodel.dart | 166 +++- app/lib/views/about_dialog.dart | 26 + app/lib/views/pass_list/pass_list_item.dart | 2 + app/lib/views/pass_list/pass_list_panel.dart | 4 + .../pass_settings/pass_settings_inline.dart | 133 ++- app/test/attribution_test.dart | 87 +- app/test/descale_detect_widget_test.dart | 112 +++ app/test/filter_schema_curation_test.dart | 4 + app/test/gpu_plugin_notice_widget_test.dart | 124 +++ app/test/gpu_plugin_status_test.dart | 125 +++ app/test/integration_ai_upscale_test.dart | 348 ++++++++ app/test/integration_descale_test.dart | 209 +++++ .../integration_filter_parameters_test.dart | 93 +++ app/test/integration_frame_mapping_test.dart | 18 +- app/test/integration_gpu_plugins_test.dart | 206 +++++ ...tegration_high_bit_depth_filters_test.dart | 26 + app/test/integration_new_passes_test.dart | 43 + app/test/integration_rife_test.dart | 143 ---- ...test.dart => knlm_notice_widget_test.dart} | 54 +- app/test/pass_list_stages_test.dart | 1 + app/test/rife_status_test.dart | 75 -- app/test/support/gpu_expectations.dart | 77 ++ app/test/vapoursynth_integration_test.dart | 2 +- docs/ENGINEERING_NOTES.md | 136 ++++ licenses/NOTICES.txt | 65 +- worker/src/dependency_locator.rs | 144 +++- worker/src/gpu_plugin.rs | 765 ++++++++++++++++++ worker/src/lib.rs | 3 +- worker/src/main.rs | 125 ++- worker/src/models/crop_resize_parameters.rs | 91 +++ worker/src/models/descale_parameters.rs | 179 ++++ worker/src/models/frame_rate_parameters.rs | 2 +- worker/src/models/mod.rs | 2 + .../src/models/noise_reduction_parameters.rs | 153 ++++ worker/src/models/processing_pipeline.rs | 45 +- worker/src/native_res.rs | 329 ++++++++ worker/src/pipeline_executor.rs | 13 +- worker/src/rife.rs | 582 ------------- worker/src/script_generator.rs | 179 +++- worker/templates/pipeline_template.vpy | 147 +++- worker/templates/preview_template.vpy | 147 +++- worker/tests/filter_integration_test.rs | 318 +++++++- 65 files changed, 6399 insertions(+), 1245 deletions(-) create mode 100755 .github/scripts/provision-vulkan.sh create mode 100644 app/assets/filters/core/descale.json create mode 100644 app/lib/models/descale_parameters.dart create mode 100644 app/lib/models/gpu_plugin_status.dart create mode 100644 app/lib/models/native_res_report.dart delete mode 100644 app/lib/models/rife_status.dart create mode 100644 app/test/descale_detect_widget_test.dart create mode 100644 app/test/gpu_plugin_notice_widget_test.dart create mode 100644 app/test/gpu_plugin_status_test.dart create mode 100644 app/test/integration_ai_upscale_test.dart create mode 100644 app/test/integration_descale_test.dart create mode 100644 app/test/integration_gpu_plugins_test.dart delete mode 100644 app/test/integration_rife_test.dart rename app/test/{rife_notice_widget_test.dart => knlm_notice_widget_test.dart} (54%) delete mode 100644 app/test/rife_status_test.dart create mode 100644 app/test/support/gpu_expectations.dart create mode 100644 worker/src/gpu_plugin.rs create mode 100644 worker/src/models/descale_parameters.rs create mode 100644 worker/src/native_res.rs delete mode 100644 worker/src/rife.rs diff --git a/.github/scripts/provision-vulkan.sh b/.github/scripts/provision-vulkan.sh new file mode 100755 index 00000000..869c8594 --- /dev/null +++ b/.github/scripts/provision-vulkan.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# Give a CI runner a Vulkan device, so the ncnn-on-Vulkan features (RIFE, and +# Real-ESRGAN / Real-CUGAN upscaling) are actually exercised rather than skipped. +# +# The hosted runners have no GPU. Linux and Windows get Mesa's lavapipe, a +# Vulkan driver that runs on the CPU — slow, but the same plugin, model and +# script path a user's GPU takes, and ncnn accepts a CPU device. macOS arm64 +# needs nothing: the bundle's MoltenVK runs on the runner's virtual Metal GPU. +# +# Wherever a device is provisioned this sets VAPOURBOX_EXPECT_VULKAN=1, and the +# GPU tests then FAIL rather than skip if a feature reports itself unavailable +# (app/test/support/gpu_expectations.dart). On a platform that does not ship +# the features (macos-x64, linux-arm64) nothing is provisioned and the tests +# assert the "not available for this platform" answer instead. +# +# Usage: provision-vulkan.sh (macos-arm64, windows-x64, linux-x64, …) +set -euo pipefail + +platform="${1:?usage: provision-vulkan.sh }" + +# Windows: the Khronos loader from LunarG's runtime components and lavapipe from +# mesa-dist-win (MIT), each pinned by hash. Keep these with the README's note. +VKRT_VERSION="1.4.328.1" +VKRT_SHA256="293efbb2c11e60429df7770f40e3c5c811a7fa42af81de0cc41e8718ddb2b373" +MESA_VERSION="26.2.2" +MESA_SHA256="0f21394d44036019bb5c07a3788dd278fd7b7b3469077a4686f0d639e8e874a1" + +case "$platform" in + linux-x64) + sudo apt-get update -qq + sudo apt-get install -y -qq --no-install-recommends mesa-vulkan-drivers libvulkan1 vulkan-tools + vulkaninfo --summary || true + echo "VAPOURBOX_EXPECT_VULKAN=1" >> "$GITHUB_ENV" + ;; + windows-x64) + dir="$RUNNER_TEMP/vulkan" + rm -rf "$dir" && mkdir -p "$dir" + curl -fsSL -o "$dir/vkrt.zip" \ + "https://sdk.lunarg.com/sdk/download/$VKRT_VERSION/windows/VulkanRT-X64-$VKRT_VERSION-Components.zip" + echo "$VKRT_SHA256 $dir/vkrt.zip" | sha256sum -c - + unzip -q -j "$dir/vkrt.zip" \ + "VulkanRT-X64-$VKRT_VERSION-Components/x64/vulkan-1.dll" \ + "VulkanRT-X64-$VKRT_VERSION-Components/x64/vulkaninfo.exe" -d "$dir" + curl -fsSL -o "$dir/mesa.7z" \ + "https://github.com/pal1000/mesa-dist-win/releases/download/$MESA_VERSION/mesa3d-$MESA_VERSION-release-msvc.7z" + echo "$MESA_SHA256 $dir/mesa.7z" | sha256sum -c - + "/c/Program Files/7-Zip/7z.exe" x -y -o"$dir/mesa" "$dir/mesa.7z" \ + x64/vulkan_lvp.dll x64/lvp_icd.x86_64.json >/dev/null + icd="$(cygpath -w "$dir/mesa/x64/lvp_icd.x86_64.json")" + # The loader reads VK_DRIVER_FILES; older loaders read VK_ICD_FILENAMES. + echo "VK_DRIVER_FILES=$icd" >> "$GITHUB_ENV" + echo "VK_ICD_FILENAMES=$icd" >> "$GITHUB_ENV" + # vulkan-1.dll on PATH, where the plugins' LoadLibrary finds it. + cygpath -w "$dir" >> "$GITHUB_PATH" + VK_DRIVER_FILES="$icd" "$dir/vulkaninfo.exe" --summary || true + echo "VAPOURBOX_EXPECT_VULKAN=1" >> "$GITHUB_ENV" + ;; + macos-arm64) + echo "VAPOURBOX_EXPECT_VULKAN=1" >> "$GITHUB_ENV" + ;; + *) + echo "No GPU features ship on $platform; the tests assert that answer instead." + ;; +esac diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index 8542a5ea..05099d46 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -100,6 +100,11 @@ jobs: ln -sfn "$BP/opt/whisper-cpp/bin" addons/whisper/bin ln -sfn "$BP/opt/whisper-cpp/lib" addons/whisper/lib + - name: Provision Vulkan (RIFE, AI upscaling) + # Lavapipe on the GPU-less Linux/Windows runners; sets + # VAPOURBOX_EXPECT_VULKAN=1 so the GPU tests must run, not skip. + run: bash .github/scripts/provision-vulkan.sh macos-${{ matrix.arch }} + - name: Rust tests # Debug build: DependencyLocator's upward search then finds the repo-root # deps/ and addons/ (release mode looks under ~/Library). @@ -172,6 +177,12 @@ jobs: cp "$(dirname "$CLI")"/*.dll addons/whisper/ 2>/dev/null || true ls addons/whisper/ + - name: Provision Vulkan (RIFE, AI upscaling) + # Lavapipe on the GPU-less Linux/Windows runners; sets + # VAPOURBOX_EXPECT_VULKAN=1 so the GPU tests must run, not skip. + shell: bash + run: bash .github/scripts/provision-vulkan.sh windows-x64 + - name: Rust tests run: cd worker && cargo test -- --nocapture @@ -260,6 +271,11 @@ jobs: ln -sfn "${{ github.workspace }}/deps" "$HOME/.local/share/VapourBox/deps" ln -sfn "${{ github.workspace }}/addons" "$HOME/.local/share/VapourBox/addons" + - name: Provision Vulkan (RIFE, AI upscaling) + # Lavapipe on the GPU-less Linux/Windows runners; sets + # VAPOURBOX_EXPECT_VULKAN=1 so the GPU tests must run, not skip. + run: bash .github/scripts/provision-vulkan.sh linux-x64 + - name: Rust tests run: cd worker && cargo test -- --nocapture diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 56f40ab3..bb40309f 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -109,6 +109,11 @@ jobs: ln -sfn "$BP/opt/whisper-cpp/bin" addons/whisper/bin ln -sfn "$BP/opt/whisper-cpp/lib" addons/whisper/lib + - name: Provision Vulkan (RIFE, AI upscaling) + # Lavapipe on the GPU-less Linux/Windows runners; sets + # VAPOURBOX_EXPECT_VULKAN=1 so the GPU tests must run, not skip. + run: bash .github/scripts/provision-vulkan.sh macos-${{ matrix.arch }} + - name: Build worker (debug) run: cd worker && cargo build @@ -189,6 +194,12 @@ jobs: cp "$(dirname "$CLI")"/*.exe addons/whisper/ 2>/dev/null || true cp "$(dirname "$CLI")"/*.dll addons/whisper/ 2>/dev/null || true + - name: Provision Vulkan (RIFE, AI upscaling) + # Lavapipe on the GPU-less Linux/Windows runners; sets + # VAPOURBOX_EXPECT_VULKAN=1 so the GPU tests must run, not skip. + shell: bash + run: bash .github/scripts/provision-vulkan.sh windows-x64 + - name: Build worker (debug) run: cd worker && cargo build @@ -273,6 +284,11 @@ jobs: ln -sfn "${{ github.workspace }}/deps" "$HOME/.local/share/VapourBox/deps" ln -sfn "${{ github.workspace }}/addons" "$HOME/.local/share/VapourBox/addons" + - name: Provision Vulkan (RIFE, AI upscaling) + # Lavapipe on the GPU-less Linux/Windows runners; sets + # VAPOURBOX_EXPECT_VULKAN=1 so the GPU tests must run, not skip. + run: bash .github/scripts/provision-vulkan.sh linux-x64 + - name: Build worker (debug) run: cd worker && cargo build diff --git a/CLAUDE.md b/CLAUDE.md index abdfdcdc..c9c4a4d8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -102,7 +102,8 @@ VapourBox/ | `worker/src/pixel_format.rs` | Source `pix_fmt` → pipe format (see "Source Pixel Formats") | | `worker/src/cpu.rs` | The x86 deps CPU tier (`v3`/`v2`) — the single decision; see "CPU tiers" | | `worker/src/source_decode.rs` | Decoder ffmpeg args for encode + preview: full stored frame, size-guarded (see "Source Frame Size") | -| `worker/src/rife.rs` | RIFE's file layout, its `--probe-rife` probe, and the user-facing reason/remedy when it can't run (see "RIFE Frame Interpolation") | +| `worker/src/gpu_plugin.rs` | The ncnn-on-Vulkan plugins (RIFE, AI upscaling): file layout, the `--probe-rife` / `--probe-upscale` probe, and the user-facing reason/remedy when they can't run (see "GPU Plugins") | +| `worker/src/native_res.rs` | `--detect-native`: the resolution and kernel a source was upscaled from, for the Descale pass (see "Descale") | | `worker/templates/pipeline_template.vpy` | VapourSynth script template | | `worker/tests/filter_integration_test.rs` | Filter integration tests | @@ -117,7 +118,8 @@ VapourBox/ | `app/lib/services/preset_service.dart` | Save/load user presets | | `app/lib/services/temp_directory_service.dart` | Configurable scratch-file directory (see "Temporary Files Directory") | | `app/lib/services/overwrite_behavior_service.dart` | Default action for existing output files (see "Existing Output Files") | -| `app/lib/models/rife_status.dart` | The worker's RIFE probe result, and the Frame Rate panel's notice decision | +| `app/lib/models/native_res_report.dart` | What `--detect-native` found, as the Descale panel's Detect shows and applies it | +| `app/lib/models/gpu_plugin_status.dart` | The worker's GPU-plugin probe results, and the panels' notice decision | | `app/assets/filters/core/*.json` | Built-in filter schema definitions | ## Build Commands @@ -1050,48 +1052,146 @@ asserted in `shutdown_when_done_test.dart`: extension. Use `colorScheme` roles, not fixed colours, for UI chrome — fixed black/white only belongs on overlays drawn over video. -### RIFE Frame Interpolation - -RIFE 4.26 (`rife.RIFE`, ncnn on Vulkan) is the Frame Rate pass's third method, -`FrameRateMethod::Rife` (`"rife"`). It is the only method that needs more than -the deps bundle, and the rules all follow from one fact: **without a usable -Vulkan implementation the plugin segfaults** (in `ncnn::create_gpu_instance`) -instead of raising. - -- **Never autoloaded.** The plugin, the model and (macOS) MoltenVK live in - `deps//vapoursynth/rife/`, outside the plugin directory; the - template `LoadPlugin`s it inside `{{#FRAME_RATE_RIFE}}` only. A missing - runtime DLL, a broken driver or AVX in a static initializer can therefore - only affect a RIFE job. Don't move it into `plugins/` / `vs-plugins/`. +### GPU Plugins: RIFE and AI Upscaling + +Two features run a neural network through ncnn on Vulkan: **RIFE** 4.26 +(`rife.RIFE`), the Frame Rate pass's `FrameRateMethod::Rife`, and **AI +upscaling** (vs-mlrt's VSNCNN, `ncnn.Model`, with Real-ESRGAN's anime video +model or Real-CUGAN), Crop & Resize's `UpscaleMethod::RealEsrgan` / +`RealCugan`. `worker/src/gpu_plugin.rs` (`GpuPlugin::{Rife, Upscale}`) holds +everything they share, and the rules all follow from one fact: **without a +usable Vulkan implementation ncnn can segfault** (RIFE does, in +`ncnn::create_gpu_instance`; VSNCNN raises "get_gpu_device failed") rather +than fail cleanly. + +- **Never autoloaded.** RIFE and its model live in `vapoursynth/rife/`, VSNCNN + and its models in `vapoursynth/ncnn/`, and on macOS the MoltenVK both use in + `vapoursynth/moltenvk/` — all outside the plugin directory. The templates + `LoadPlugin` them inside `{{#FRAME_RATE_RIFE}}` / `{{#UPSCALE_AI}}` only, so + a missing runtime DLL, a broken driver or AVX in a static initializer can + only affect a job that uses them. Don't move them into `plugins/` / + `vs-plugins/`. - **macOS preloads MoltenVK** (`ctypes.CDLL(..., RTLD_GLOBAL)`) before the - first call: the dylibs leave `vkGetInstanceProcAddr` to whatever is already - in the process. Without it every RIFE call is a SIGSEGV. -- **One probe, one wording.** `rife.rs` prechecks the platform and files, then - runs the plugin once in a throwaway vspipe, printing stage markers so a + first call: both dylibs leave `vkGetInstanceProcAddr` to whatever is already + in the process. +- **One probe, one wording.** `gpu_plugin.rs` prechecks the platform and files, + then runs the plugin once in a throwaway vspipe, printing stage markers so a silent crash still says where it died, and classifies the result into a - `RifeReason` with a user-facing remedy (`RifeStatus::message`). The app shows - that text when RIFE is selected (`--probe-rife`, `rifeNoticeFor`, with a - *Check again* button); the worker refuses a RIFE job with the same text - (`ScriptGenerator::with_rife`, `ensure_rife_ready`) rather than emit a script - that would crash. Only a **positive** result is cached per boot — a cached - failure would keep blaming a driver the user has just installed. - `VAPOURBOX_DISABLE_VULKAN=1` forces the unavailable path. -- **Platforms:** macos-arm64, windows-x64 (both tiers), linux-x64 (both tiers). - `rife::platform_ships_rife` and `_extraFiles` in `deps-expected-plugins.json` - must agree (`attribution_test.dart`). macOS x64's prebuilt targets macOS 13 - and fails `check-load-time-simd.py`; linux-arm64 has no upstream build. Both - need a source build before they can be added. -- **Real frames pass through untouched.** RIFE works in RGBS; every output + `GpuPluginReason` with a user-facing remedy (`GpuPluginStatus::message`, + naming the feature). The app shows that text the moment the feature is + selected (`--probe-rife` / `--probe-upscale`, `gpuPluginSelectedBy`, + `gpuPluginNoticeFor`, with a *Check again* button); the worker refuses such + a job with the same text (`ScriptGenerator::with_rife` / `with_upscale`, + `ensure_methods_ready`) rather than emit a script that would crash. Only a + **positive** result is cached per boot (`rife-probe.json`, + `ncnn-probe.json`) — a cached failure would keep blaming a driver the user + has just installed. `VAPOURBOX_DISABLE_VULKAN=1` forces the unavailable path. +- **Platforms:** macos-arm64, windows-x64 (both tiers), linux-x64 (both tiers), + for both. `gpu_plugin::platform_ships_rife` / `platform_ships_upscale` and + `_extraFiles` in `deps-expected-plugins.json` must agree + (`attribution_test.dart`). macOS x64 has no MoltenVK copy or usable RIFE + prebuilt; linux-arm64 has no upstream RIFE and Tencent publishes no aarch64 + Linux ncnn. +- **RIFE: real frames pass through untouched.** It works in RGBS; every output frame it would merely copy (an exact source position, the held tail) is - taken from the source by `FrameEval` instead, so photographed frames are - bit-exact rather than chroma-softened by a YUV→RGB→YUV round trip. The count - is `floor(n * num / den)` of the reduced ratio — exactly `FrameMap::Retime`. - `sc=True` with `misc.SCDetect` holds the frame across a cut rather than - blending two shots. -- The model is the ncnn conversion committed to the plugin's repo, fetched at - a pinned commit and checked by SHA-256 in every `download-deps-*` script. - -See docs/ENGINEERING_NOTES.md (2026-10-24) for the probe measurements. + taken from the source by `FrameEval`, so photographed frames are bit-exact. + The count is `floor(n * num / den)` of the reduced ratio — exactly + `FrameMap::Retime`. `sc=True` with `misc.SCDetect` holds the frame across a + cut rather than blending two shots. +- **Upscaling: models, factors, tiles.** Five of vs-mlrt's ONNX conversions + ship (`gpu_plugin::UPSCALE_MODELS`): Real-ESRGAN animevideo 2x/4x and + Real-CUGAN 2x at three strengths. Factors follow the edge-directed path's + rule (3x behaves as 2x); CUGAN 4x is its 2x model twice, denoising on the + first pass only (`CropResizeParameters::ai_upscale_models`). Above SD the + frame runs in 512-px tiles with 16 px overlap — whole-frame CUGAN at 1080p + ran out of GPU memory on a 16 GB Mac. Float RGB in and out with the source's + matrix/range, odd sizes edge-padded, frame props copied back, fp16 (≤0.3/255 + from fp32, ~3x faster). Real-ESRGAN measurably tints flat colour (up to ~11 + levels at 4x, in fp32 too); that is the model, not the pipeline. +- **VSNCNN is built from source on macOS and Linux** (only Windows has a + published build): static protobuf v3.21.12 and onnx at vs-mlrt's pinned + commit, Tencent's prebuilt ncnn 20260526, VapourSynth's API 3 headers from + the R78 tag. Models come from vs-mlrt's ~810 MB `models.vX.7z` (the only + place they are published), extracted in one pass and each checked by hash. + Every pin and hash is in all three `download-deps-*` scripts. +- **Tested for real in CI.** The hosted runners have no GPU, so + `.github/scripts/provision-vulkan.sh` gives Linux and Windows Mesa's + lavapipe (a CPU Vulkan driver ncnn accepts) and sets + `VAPOURBOX_EXPECT_VULKAN=1` wherever a device exists (macOS arm64 uses the + runner's Metal GPU through MoltenVK). Under that variable every GPU test + **fails** rather than skips if a feature reports itself unavailable + (`app/test/support/gpu_expectations.dart`), so a broken bundle cannot hide + behind "no GPU". `integration_gpu_plugins_test.dart` (push gate) probes, + checks the refusal and the script, and renders a real frame per feature; + `integration_ai_upscale_test.dart` and the RIFE group of + `integration_frame_mapping_test.dart` (heavy) encode and check pixels — the + upscalers against Spline36 on a synthetic pattern (colour within 2 levels for + CUGAN 2x, bar centre within 0.75 px for all). + +See docs/ENGINEERING_NOTES.md (2026-10-24, 2026-10-25) for the measurements. + +### Descale + +The Descale pass (`DescaleParameters`, `PassType::Descale`, app 1.5.0) inverts +a studio's upscale on luma with `descale.De*` (r12, MIT, a PyPI wheel on every +platform, autoloaded — its AVX2 kernel is picked at run time), re-upscales +with NNEDI3 (`_nnedi3()`, doubling to at least the source size, then one +Spline36 resample that also folds in the doublings' half-pixel shifts), and +keeps chroma and the source size. Rules: + +- **It runs right after SpotLess, before Noise Reduction** — descale needs the + upscale exactly as the studio made it. `test_177` asserts the order. +- **Protect credits and text** (`protectDetail`, default on): where + re-applying the kernel to the descaled picture misses the source by more + than `detailThreshold` 8-bit levels, the source is kept (binarised, grown + twice, softened). On a never-upscaled picture that is nearly every pixel — + correct, and why the parity test switches it off. +- **Detect** (`--detect-native`, `native_res.rs`) is what makes the pass + usable: getnative's method, with these measured traps. Scan every height + from half the source's, with one kernel per family (bilinear, Catmull-Rom, + Lanczos 3, Spline36 — a master only dips under its own); only heights with + neighbours on *both* sides are candidates (the error slopes down toward the + source height and a one-sided window reads that as a dip); cap each pixel's + error at 2 levels before averaging, or credits swamp the dip; rank 12 + kernels at the best height over four sampled frames. `found` only when the + dip is ≤ 0.5 of its neighbours' median — a never-upscaled VHS capture + measures ~0.9, a credit-free master ~0.14. All candidates are spliced into + one clip and read with `frames()`, so VapourSynth runs them in parallel + (~30 s at 1080p). A fixture for any of this must have real detail at its + native resolution: an upscale of low-res footage is so smooth that 8-bit + rounding hides every height's misfit. +- The panel never applies a guess: no clear dip is reported as such. + `integration_descale_test.dart` (heavy) checks detection on known masters, + protection (grid exact vs. destroyed without the mask) and preview = encode. + +### BM3D and KNLMeans Noise Reduction + +Two advanced-only Noise Reduction methods (`bm3d`, `knlmeans`, app 1.5.0; +the simple slots are full at four). + +- **BM3D works in 16-bit, one GRAY plane at a time.** `VAggregate` always + returns 16-bit and `VFinal` needs its basic estimate in its input's format, + so the block converts up and restores the source format with dither. BM3D + refuses subsampled chroma, so each plane is its own GRAY clip: 4:2:0 chroma + is denoised at its own resolution rather than resampled to 4:4:4 and back. + `sigma` is in 8-bit units at every depth (parity 0.36/255 at 10-bit). +- **BM3D ships everywhere from deps 1.14.0**, pinned to r12 (`BM3D_TAG` on + macOS/Linux = PyPI wheel 12.0 on Windows). Before, only macOS arm64 had it: + Linux's `patch_meson_vs_include` missed r12's nested + `include_directories('include', run_command(py, '-c', 'import vapoursynth…'))`, + and macOS x64 never ran `build_plugin`. Both failed silently because the + plugin wasn't in their `deps-expected-plugins.json` lists. +- **KNLMeans has its own device probe**, separate from `knlm_available`. + havsfunc calls KNLMeansCL with the default `device_type="auto"`, which fails + on Apple Silicon (`CL_INVALID_VALUE`) where `"gpu"` works. So `knlm` still + answers for QTGMC's call, and `DependencyLocator::knlm_device` (gpu, then + auto; `knlm-device-probe.json`) is the device our own call passes. With no + device the job is refused with `KNLM_NO_DEVICE_MESSAGE`, the same text the + panel shows (`knlmDevice`/`knlmMessage` from `--probe-opencl`), through + `ScriptGenerator::with_knlm` / `ensure_methods_ready`, as for RIFE. +- **The Noise Reduction pass's preview radius is per method** + (`NoiseReductionParameters::temporal_radius`): BM3D's Final step re-reads + its basic estimate's neighbours, so it reaches 2r; a fixed 3 was too few. ### Preset System @@ -1984,4 +2084,4 @@ Full write-ups (root causes, measurements) for each entry are in | 1.11.0 | 2026-09 | **Issue #92**: every x86 bundle ships in two **CPU tiers** — `v3` (x86-64-v3; the plain asset names) and `v2` (`…-x64-v2`, anything older), chosen by the app from `vapourbox-worker --probe-cpu`. Each bundle carries one autoloaded zsmooth (replacing 1.10.0's two-builds-loaded-by-path). macOS v2 builds MVTools v24 from source without its AVX2 files, whose static initializers SIGILLed inside `dlopen` on pre-AVX Macs. v2 bundles are gated (SDE on Linux/Windows, static check on macOS) before publishing | | 1.12.0 | 2026-10-04 | **Issue #101**: akarin 1.4.1 → **1.5.0** on every platform. The 1.4.1 Linux wheel put JIT code on the `brk` heap, where SELinux denies `PROT_EXEC`, so every job segfaulted on Fedora/RHEL from deps 1.8.0 on. `download-deps-linux.sh` now fails if `libakarin.so` lacks the mmap allocator | | 1.13.0 | 2026-10 | One new plugin; every other binary is built from the same sources as 1.12.0. Adds **vszip** 22.1.0 (MIT) for `vszip.CLAHE`, Color Correction's local contrast (issue #90). Baseline build only, flat in the plugin directory on every platform (the wheels' AVX2/AVX-512 variants are left out); macOS x64 builds the sdist with Zig 0.16.0 for the 12.0 floor, with header padding so it can be signed. The v2 CPU tier is no longer a second full bundle: each x64 platform publishes its v3 bundle plus `…-x64-v2-delta.zip`, the files in `_tierFiles` (zsmooth; on macOS also MVTools), extracted over it (`"tierFormat": "delta"`). Every x64 machine downloads the same bundle; only an older CPU fetches the few MB more. This is also what lets the Linux AppImage carry its deps for both tiers without doubling in size | -| 1.14.0 | unreleased | Adds **RIFE** 4.26 (`r9_mod_v33`, MIT) + its model in `vapoursynth/rife/` on macos-arm64, windows-x64 and linux-x64, and **MoltenVK** 1.4.2 (Apache-2.0) beside it on macOS. Not autoloaded; see "RIFE Frame Interpolation" | +| 1.14.0 | unreleased | Adds **RIFE** 4.26 (`r9_mod_v33`, MIT) + its model in `vapoursynth/rife/`, and **VSNCNN** (vs-mlrt v15.16, GPL-3.0) + Real-ESRGAN / Real-CUGAN models in `vapoursynth/ncnn/`, on macos-arm64, windows-x64 and linux-x64, with **MoltenVK** 1.4.2 (Apache-2.0) in `vapoursynth/moltenvk/` on macOS. Not autoloaded; see "GPU Plugins". **BM3D** r12 (MIT) on every platform; it had been macOS arm64 only (see "BM3D and KNLMeans Noise Reduction"). **descale** r12 (MIT) on every platform, for the Descale pass | diff --git a/README.md b/README.md index 873a2b2d..6a4ba3de 100644 --- a/README.md +++ b/README.md @@ -85,7 +85,7 @@ GPU-accelerated deinterlacing (NNEDI3CL) needs your GPU's OpenCL driver installe -**RIFE frame interpolation** (in the Frame Rate pass) runs on the GPU through Vulkan. It works on any Apple Silicon Mac, and on Windows and Linux x64 PCs with a current graphics driver; Windows also needs the Microsoft Visual C++ Redistributable. It isn't available on Intel Macs or Linux on ARM yet. When you choose it, VapourBox checks that it can run and, if it can't, says what is missing and how to fix it. +**RIFE frame interpolation** (in the Frame Rate pass) and **AI upscaling** with Real-ESRGAN or Real-CUGAN (in Crop & Resize) run on the GPU through Vulkan. They work on any Apple Silicon Mac, and on Windows and Linux x64 PCs with a current graphics driver; RIFE on Windows also needs the Microsoft Visual C++ Redistributable. They aren't available on Intel Macs or Linux on ARM yet. When you choose one, VapourBox checks that it can run and, if it can't, says what is missing and how to fix it. **Older Intel and AMD processors** (before about 2013 — no AVX2) are supported on all three platforms. On first launch VapourBox checks the processor and installs the matching set of processing components; a machine moved to a different CPU gets the right set on its next launch. The older-CPU set runs some filters more slowly, never differently. Every build of that set is checked for pre-AVX processors (e.g. a 2010 Mac Pro) on macOS and Linux; on Windows it is checked for processors with AVX but no AVX2, and older ones are expected to work but can't be tested automatically. @@ -103,7 +103,7 @@ GPU-accelerated deinterlacing (NNEDI3CL) needs your GPU's OpenCL driver installe ## The filter pipeline -Twenty-one filters, each switchable independently. Most sources need none or a few. They always run in the order below, top to bottom, whichever ones are switched on — the same order the list in the app shows them in. The order cannot be rearranged. +Twenty-two filters, each switchable independently. Most sources need none or a few. They always run in the order below, top to bottom, whichever ones are switched on — the same order the list in the app shows them in. The order cannot be rearranged. | # | Filter | What it addresses | |---|--------|-------------------| @@ -113,21 +113,22 @@ Twenty-one filters, each switchable independently. Most sources need none or a f | 4 | **Deflicker** | Brightness pulsing between frames, which is what scanned cine film almost always has. | | 5 | **DeScratch** | Vertical scratch lines on scanned film. | | 6 | **SpotLess** | Dust, dirt and single-frame specks. RemoveDirt is the faster choice — around six times the speed for about 60% of the removal. | -| 7 | **Noise Reduction** | Grain and video noise across the whole frame. Motion-compensated by default, with mClean as a gentler alternative that keeps more detail; DFTTest, FFT3DFilter, TTempSmooth, FluxSmooth, STPresso, TemporalDegrain2 and a large-window median are available under advanced options for noise the default handles badly. | -| 8 | **Chroma Denoise** | Blotchy, smeared color — common on VHS captures and old camcorder footage. Leaves luma detail untouched. | -| 9 | **Dehalo** | Bright outlines around edges, ringing, and residual ghosting left by a deinterlacer. HQDeringmod targets ringing specifically. | -| 10 | **Deblock** | Square blocking from heavy compression, and the ringing around edges that comes with it. | -| 11 | **Deband** | Visible steps in gradients and skies. | -| 12 | **Anti-Aliasing** | Stair-stepping on diagonal edges, left by deinterlacing or upscaling. Runs before sharpening, which would otherwise make the steps more visible. | -| 13 | **Chroma Fixes** | Colour that sits sideways from the picture (corrected automatically or by hand), bleeding past edges, rainbowing and dot crawl — including the shimmering kind that only shows when the picture moves — and residual combing. Each repair has its own switch, and its settings appear only once it is on. | -| 14 | **Color Correction** | Brightness, contrast, saturation, hue, levels, white balance (warm/cool, green/magenta), lifting detail out of the shadows of underexposed footage, and lifting local contrast across a dim or flat picture. Levels and white balance can each be measured automatically or set by hand. | -| 15 | **Stabilize** | Shake and weave — telecine wobble, jittery film scans, handheld footage. Runs last before cropping, so a small crop removes the edges it exposes. | -| 16 | **Rotate / Flip** | Footage shot sideways, mirrored captures, scans that came off the scanner the wrong way round. | -| 17 | **Crop & Resize** | Trimming overscan, scaling, and edge-directed upscaling — plus bars in a colour of your choice to bring a cropped picture back to an exact frame size (720×576 for PAL DVD, 720×480 for NTSC) without rescaling it. | -| 18 | **Sharpen** | Soft sources needing edge and fine detail recovery. aWarpSharp2 sharpens by warping edges instead of raising contrast, so it adds no halos. Runs after the resize, on the picture that is actually delivered, and before any added grain. | -| 19 | **Film Grain** | Grain added back after denoising, so the picture is not left plastic — and to hide banding in skies and fades. | -| 20 | **Frame Rate** | Converting between PAL and NTSC rates, for a tape that was already converted once and now plays at the wrong speed. New frames come from motion interpolation, from RIFE (an AI interpolator that runs on the GPU), or from plain frame repetition. | -| 21 | **Subtitles** | Whisper AI speech-to-text — written alongside the video as `.srt`, embedded as a selectable track, burnt into the picture, or a combination. | +| 7 | **Descale** | Soft, haloed line art in HD anime and other video drawn below its delivery resolution and upscaled by the studio. Inverts that upscale exactly and upscales again with NNEDI3; Detect finds the resolution and kernel the studio used, and says so when a source was never upscaled. Credits and text drawn at full resolution are left untouched. | +| 8 | **Noise Reduction** | Grain and video noise across the whole frame. Motion-compensated by default, with mClean as a gentler alternative that keeps more detail; DFTTest, FFT3DFilter, TTempSmooth, FluxSmooth, STPresso, TemporalDegrain2, BM3D, KNLMeansCL (GPU) and a large-window median are available under advanced options for noise the default handles badly. BM3D is the usual choice for anime: it cleans flat colour without smearing line art. KNLMeansCL needs a graphics card with working OpenCL; VapourBox says so when you pick it if yours has none. | +| 9 | **Chroma Denoise** | Blotchy, smeared color — common on VHS captures and old camcorder footage. Leaves luma detail untouched. | +| 10 | **Dehalo** | Bright outlines around edges, ringing, and residual ghosting left by a deinterlacer. HQDeringmod targets ringing specifically. | +| 11 | **Deblock** | Square blocking from heavy compression, and the ringing around edges that comes with it. | +| 12 | **Deband** | Visible steps in gradients and skies. | +| 13 | **Anti-Aliasing** | Stair-stepping on diagonal edges, left by deinterlacing or upscaling. Runs before sharpening, which would otherwise make the steps more visible. | +| 14 | **Chroma Fixes** | Colour that sits sideways from the picture (corrected automatically or by hand), bleeding past edges, rainbowing and dot crawl — including the shimmering kind that only shows when the picture moves — and residual combing. Each repair has its own switch, and its settings appear only once it is on. | +| 15 | **Color Correction** | Brightness, contrast, saturation, hue, levels, white balance (warm/cool, green/magenta), lifting detail out of the shadows of underexposed footage, and lifting local contrast across a dim or flat picture. Levels and white balance can each be measured automatically or set by hand. | +| 16 | **Stabilize** | Shake and weave — telecine wobble, jittery film scans, handheld footage. Runs last before cropping, so a small crop removes the edges it exposes. | +| 17 | **Rotate / Flip** | Footage shot sideways, mirrored captures, scans that came off the scanner the wrong way round. | +| 18 | **Crop & Resize** | Trimming overscan, scaling, and upscaling — edge-directed, or with Real-ESRGAN or Real-CUGAN, AI upscalers trained on anime that run on the GPU — plus bars in a colour of your choice to bring a cropped picture back to an exact frame size (720×576 for PAL DVD, 720×480 for NTSC) without rescaling it. | +| 19 | **Sharpen** | Soft sources needing edge and fine detail recovery. aWarpSharp2 sharpens by warping edges instead of raising contrast, so it adds no halos. Runs after the resize, on the picture that is actually delivered, and before any added grain. | +| 20 | **Film Grain** | Grain added back after denoising, so the picture is not left plastic — and to hide banding in skies and fades. | +| 21 | **Frame Rate** | Converting between PAL and NTSC rates, for a tape that was already converted once and now plays at the wrong speed. New frames come from motion interpolation, from RIFE (an AI interpolator that runs on the GPU), or from plain frame repetition. | +| 22 | **Subtitles** | Whisper AI speech-to-text — written alongside the video as `.srt`, embedded as a selectable track, burnt into the picture, or a combination. | A few things about the order are worth knowing: @@ -298,8 +299,11 @@ Stuart Cameron — [stuart-cameron.com](https://stuart-cameron.com) - **dubhater** — a long list of VapourSynth ports this app relies on: MVTools, nnedi3, AWarpSharp2, TemporalMedian, FluxSmooth, Bifrost and the `adjust` Tweak port - **HolyWu** — the VapourSynth ports of DFTTest, TTempSmooth, TCanny, CTMF, DCTFilter, Deblock, AddGrain, CAS, EEDI3 and NNEDI3CL - **mawen1250** — Retinex, BM3D and mvsfunc +- **Edoardo Brunetti** — KNLMeansCL +- **Frechdachs** (maintained by Jaded Encoding Thaumaturgy) — descale - **vszip** by Julek — CLAHE, the local-contrast lift in Color Correction - **RIFE** by hzwer (with Megvii) — the frame interpolation network, run through HolyWu's VapourSynth plugin (built on nihui's rife-ncnn-vulkan and Tencent's ncnn), with **MoltenVK** for Vulkan on macOS +- **Real-ESRGAN** by Xintao Wang and **Real-CUGAN** by bilibili — the AI upscaling networks, run through AmusementClub's **vs-mlrt** (VSNCNN) - **whisper.cpp** by the ggml authors — speech recognition for subtitle generation - **libdvdread** by VideoLAN — DVD reading and navigation - **FFmpeg** project — video encoding diff --git a/Scripts/deps-expected-plugins.json b/Scripts/deps-expected-plugins.json index 496c41fc..2763a287 100644 --- a/Scripts/deps-expected-plugins.json +++ b/Scripts/deps-expected-plugins.json @@ -13,33 +13,63 @@ "vapoursynth/plugins/libzsmooth.so" ] }, - "_extraFilesComment": "Files a COMPLETE bundle must hold outside the plugin directory, as paths from the bundle root. Today that is RIFE (worker/src/rife.rs): its plugin, model and, on macOS, MoltenVK live in vapoursynth/rife/, deliberately outside the autoload directory, so the plugin-directory lists above cannot cover them. The package-deps-* scripts assert these too. A platform with no entry ships no RIFE (macos-x64, linux-arm64: no usable prebuilt), and the worker's rife::platform_ships_rife must agree — app/test/attribution_test.dart checks both directions.", + "_extraFilesComment": "Files a COMPLETE bundle must hold outside the plugin directory, as paths from the bundle root. Today those are the ncnn-on-Vulkan plugins (worker/src/gpu_plugin.rs): RIFE in vapoursynth/rife/, VSNCNN and its upscaling models in vapoursynth/ncnn/, and on macOS the MoltenVK both preload from vapoursynth/moltenvk/ — deliberately outside the autoload directory, so the plugin-directory lists above cannot cover them. The package-deps-* scripts assert these too. A platform with no entry ships neither (macos-x64, linux-arm64), and the worker's gpu_plugin::platform_ships_rife / platform_ships_upscale must agree — app/test/attribution_test.dart checks both directions.", "_extraFiles": { "macos-arm64": [ + "vapoursynth/moltenvk/libMoltenVK.dylib", "vapoursynth/rife/librife.dylib", - "vapoursynth/rife/libMoltenVK.dylib", "vapoursynth/rife/models/rife-v4.26_ensembleFalse/flownet.bin", - "vapoursynth/rife/models/rife-v4.26_ensembleFalse/flownet.param" + "vapoursynth/rife/models/rife-v4.26_ensembleFalse/flownet.param", + "vapoursynth/ncnn/libvsncnn.dylib", + "vapoursynth/ncnn/models/RealESRGANv2/RealESRGANv2-animevideo-xsx2.onnx", + "vapoursynth/ncnn/models/RealESRGANv2/RealESRGANv2-animevideo-xsx4.onnx", + "vapoursynth/ncnn/models/cugan/up2x-latest-no-denoise.onnx", + "vapoursynth/ncnn/models/cugan/up2x-latest-conservative.onnx", + "vapoursynth/ncnn/models/cugan/up2x-latest-denoise3x.onnx" ], "windows-x64": [ "vapoursynth/rife/librife.dll", "vapoursynth/rife/models/rife-v4.26_ensembleFalse/flownet.bin", - "vapoursynth/rife/models/rife-v4.26_ensembleFalse/flownet.param" + "vapoursynth/rife/models/rife-v4.26_ensembleFalse/flownet.param", + "vapoursynth/ncnn/vsncnn.dll", + "vapoursynth/ncnn/models/RealESRGANv2/RealESRGANv2-animevideo-xsx2.onnx", + "vapoursynth/ncnn/models/RealESRGANv2/RealESRGANv2-animevideo-xsx4.onnx", + "vapoursynth/ncnn/models/cugan/up2x-latest-no-denoise.onnx", + "vapoursynth/ncnn/models/cugan/up2x-latest-conservative.onnx", + "vapoursynth/ncnn/models/cugan/up2x-latest-denoise3x.onnx" ], "windows-x64-v2": [ "vapoursynth/rife/librife.dll", "vapoursynth/rife/models/rife-v4.26_ensembleFalse/flownet.bin", - "vapoursynth/rife/models/rife-v4.26_ensembleFalse/flownet.param" + "vapoursynth/rife/models/rife-v4.26_ensembleFalse/flownet.param", + "vapoursynth/ncnn/vsncnn.dll", + "vapoursynth/ncnn/models/RealESRGANv2/RealESRGANv2-animevideo-xsx2.onnx", + "vapoursynth/ncnn/models/RealESRGANv2/RealESRGANv2-animevideo-xsx4.onnx", + "vapoursynth/ncnn/models/cugan/up2x-latest-no-denoise.onnx", + "vapoursynth/ncnn/models/cugan/up2x-latest-conservative.onnx", + "vapoursynth/ncnn/models/cugan/up2x-latest-denoise3x.onnx" ], "linux-x64": [ "vapoursynth/rife/librife.so", "vapoursynth/rife/models/rife-v4.26_ensembleFalse/flownet.bin", - "vapoursynth/rife/models/rife-v4.26_ensembleFalse/flownet.param" + "vapoursynth/rife/models/rife-v4.26_ensembleFalse/flownet.param", + "vapoursynth/ncnn/libvsncnn.so", + "vapoursynth/ncnn/models/RealESRGANv2/RealESRGANv2-animevideo-xsx2.onnx", + "vapoursynth/ncnn/models/RealESRGANv2/RealESRGANv2-animevideo-xsx4.onnx", + "vapoursynth/ncnn/models/cugan/up2x-latest-no-denoise.onnx", + "vapoursynth/ncnn/models/cugan/up2x-latest-conservative.onnx", + "vapoursynth/ncnn/models/cugan/up2x-latest-denoise3x.onnx" ], "linux-x64-v2": [ "vapoursynth/rife/librife.so", "vapoursynth/rife/models/rife-v4.26_ensembleFalse/flownet.bin", - "vapoursynth/rife/models/rife-v4.26_ensembleFalse/flownet.param" + "vapoursynth/rife/models/rife-v4.26_ensembleFalse/flownet.param", + "vapoursynth/ncnn/libvsncnn.so", + "vapoursynth/ncnn/models/RealESRGANv2/RealESRGANv2-animevideo-xsx2.onnx", + "vapoursynth/ncnn/models/RealESRGANv2/RealESRGANv2-animevideo-xsx4.onnx", + "vapoursynth/ncnn/models/cugan/up2x-latest-no-denoise.onnx", + "vapoursynth/ncnn/models/cugan/up2x-latest-conservative.onnx", + "vapoursynth/ncnn/models/cugan/up2x-latest-denoise3x.onnx" ] }, "windows-x64": [ @@ -76,7 +106,9 @@ "VIVTC.dll", "vszip.dll", "vsznedi3.dll", - "zsmooth.dll" + "zsmooth.dll", + "bm3d.dll", + "libdescale.dll" ], "windows-x64-v2": [ "AddGrain.dll", @@ -112,7 +144,9 @@ "VIVTC.dll", "vszip.dll", "vsznedi3.dll", - "zsmooth.dll" + "zsmooth.dll", + "bm3d.dll", + "libdescale.dll" ], "macos-arm64": [ "libaddgrain.dylib", @@ -149,7 +183,8 @@ "libvivtc.dylib", "libvszip.dylib", "libznedi3.dylib", - "libzsmooth.dylib" + "libzsmooth.dylib", + "libdescale.dylib" ], "macos-x64": [ "libaddgrain.dylib", @@ -183,7 +218,9 @@ "libvivtc.dylib", "libvszip.dylib", "libznedi3.dylib", - "libzsmooth.dylib" + "libzsmooth.dylib", + "libbm3d.dylib", + "libdescale.dylib" ], "macos-x64-v2": [ "libaddgrain.dylib", @@ -217,7 +254,9 @@ "libvivtc.dylib", "libvszip.dylib", "libznedi3.dylib", - "libzsmooth.dylib" + "libzsmooth.dylib", + "libbm3d.dylib", + "libdescale.dylib" ], "linux-x64": [ "libaddgrain.so", @@ -252,7 +291,9 @@ "libvivtc.so", "libvszip.so", "libznedi3.so", - "libzsmooth.so" + "libzsmooth.so", + "libbm3d.so", + "libdescale.so" ], "linux-x64-v2": [ "libaddgrain.so", @@ -287,7 +328,9 @@ "libvivtc.so", "libvszip.so", "libznedi3.so", - "libzsmooth.so" + "libzsmooth.so", + "libbm3d.so", + "libdescale.so" ], "linux-arm64": [ "libaddgrain.so", @@ -323,6 +366,8 @@ "libvivtc.so", "libvszip.so", "libznedi3.so", - "libzsmooth.so" + "libzsmooth.so", + "libbm3d.so", + "libdescale.so" ] } diff --git a/Scripts/download-deps-linux.sh b/Scripts/download-deps-linux.sh index e4c36ce5..06a90282 100755 --- a/Scripts/download-deps-linux.sh +++ b/Scripts/download-deps-linux.sh @@ -656,6 +656,14 @@ content = re.sub( content, flags=re.DOTALL ) +# The same probe nested among other include dirs, as BM3D r12 writes it: +# include_directories('include', run_command(py, '-c', 'import vapoursynth ...')). +# Only a vapoursynth include probe is replaced; other run_command calls stay. +content = re.sub( + r\"run_command\(\s*[^,()]+,\s*'-c',\s*'import vapoursynth[^']*'[^)]*\)\.stdout\(\)\.strip\(\)\", + \"'$VS_INCLUDE_DIR'\", + content, +) with open('meson.build', 'w') as f: f.write(content) " 2>/dev/null || true @@ -666,6 +674,7 @@ build_plugin() { local repo="$2" local output_lib="$3" local build_cmd="$4" + local tag="${5:-}" # optional git tag; empty builds the default branch if [ "$FORCE" = false ] && [ -f "$PLUGINS_DIR/$output_lib" ]; then echo " $name already exists, skipping" @@ -676,7 +685,7 @@ build_plugin() { echo "=== Building $name ===" rm -rf "$name" - if ! git clone --depth 1 "$repo" "$name" 2>/dev/null; then + if ! git clone --depth 1 ${tag:+--branch "$tag"} "$repo" "$name" 2>/dev/null; then echo " Failed to clone $name" FAILED_PLUGINS+=("$name") return 1 @@ -1048,11 +1057,16 @@ build_plugin "tcanny" \ "libtcanny.so" \ "$PLUGIN_BUILD_ENV meson setup build --buildtype=release && ninja -C build" -# BM3D +# BM3D (core.bm3d - Noise Reduction's BM3D method). Pinned: keep BM3D_TAG in +# step with download-deps-macos.sh and the wheel version in +# download-deps-windows.ps1 (r12 == wheel 12.0). Links the fftw copied into +# lib/ above. +BM3D_TAG="r12" build_plugin "bm3d" \ "https://github.com/HomeOfVapourSynthEvolution/VapourSynth-BM3D.git" \ "libbm3d.so" \ - "$PLUGIN_BUILD_ENV meson setup build --buildtype=release && ninja -C build" + "$PLUGIN_BUILD_ENV meson setup build --buildtype=release && ninja -C build" \ + "$BM3D_TAG" # KNLMeansCL (OpenCL denoiser) build_plugin "knlmeanscl" \ @@ -1567,11 +1581,60 @@ else echo " vszip already exists, skipping" fi +# ============================================================================ +# descale — the Descale pass (core.descale.De*) +# ============================================================================ +# The manylinux2014 wheels (glibc 2.17, far under our floor), pinned by hash. +# Its AVX2 kernel is a separate object chosen at run time (opt=0), so nothing +# AVX runs at load and one file serves both x64 tiers. Keep DESCALE_VERSION in +# step with download-deps-{macos.sh,windows.ps1}. +DESCALE_VERSION="12" +case "$ARCH" in + x86_64) DESCALE_TAG="manylinux_2_28_x86_64" + DESCALE_SHA256="9969dca2a6feb7cd5ff2b60e9198aa529f3db02f49d5c88556b3fbcaa822f05d" ;; + aarch64) DESCALE_TAG="manylinux_2_28_aarch64" + DESCALE_SHA256="4a4b3494ac88b769333070ab279bb2d6590f37947f1669a6d628107007456793" ;; + *) DESCALE_TAG="" ;; +esac +DESCALE_OUT="$PLUGINS_DIR/libdescale.so" +echo "" +echo "=== Installing descale $DESCALE_VERSION ===" +if [ "$FORCE" = false ] && [ -f "$DESCALE_OUT" ]; then + echo " descale already exists, skipping" +else + rm -f "$DESCALE_OUT" + if [ -n "$DESCALE_TAG" ] && ( + set -e + cd "$BUILD_DIR" + rm -rf descale-whl descale.whl + url=$("$PYTHON_BIN" - "$DESCALE_VERSION" "$DESCALE_TAG" <<'PYEOF2' +import json, sys, urllib.request +ver, tag = sys.argv[1], sys.argv[2] +d = json.load(urllib.request.urlopen(f"https://pypi.org/pypi/vapoursynth-descale/{ver}/json")) +print(next(f["url"] for f in d["urls"] if f["filename"].endswith(tag + ".whl"))) +PYEOF2 +) + curl -fsSL -o descale.whl "$url" + echo "$DESCALE_SHA256 descale.whl" | sha256sum -c - + mkdir descale-whl && unzip -q descale.whl -d descale-whl + cp descale-whl/vapoursynth/plugins/libdescale.so "$DESCALE_OUT" + chmod 755 "$DESCALE_OUT" + rm -rf descale-whl descale.whl + ); then + echo " Installed descale -> libdescale.so" + BUILT_PLUGINS+=("descale") + else + rm -f "$DESCALE_OUT" + echo " Failed to install descale" + FAILED_PLUGINS+=("descale") + fi +fi + # ============================================================================ # RIFE 4.26 — the Frame Rate pass's neural interpolator (ncnn on Vulkan) # ============================================================================ # Lives in vapoursynth/rife/, NOT the autoload directory: the template loads it -# only for a RIFE job (see worker/src/rife.rs), so a machine whose GPU stack or +# only for a RIFE job (see worker/src/gpu_plugin.rs), so a machine whose GPU stack or # CPU cannot run it fails that method alone. Keep RIFE_TAG, the model commit and # the model hashes in step with download-deps-{macos.sh,windows.ps1}. # @@ -1626,6 +1689,121 @@ else fi fi +# ============================================================================ +# VSNCNN (vs-mlrt) + Real-ESRGAN / Real-CUGAN — Crop & Resize's AI upscaling +# ============================================================================ +# vs-mlrt publishes VSNCNN for Windows only; this is upstream's own +# linux-ncnn.yml recipe at the pinned tag: static protobuf and onnx from +# source, Tencent's prebuilt Ubuntu 24.04 ncnn (static, with a CMake package), +# then vsncnn's own CMakeLists. Lives in vapoursynth/ncnn/, never autoloaded +# (see worker/src/gpu_plugin.rs). x86_64 only, as for RIFE: Tencent ships no +# aarch64 Linux ncnn. Like RIFE it needs libgomp.so.1 from the system and +# dlopens the system's libvulkan.so.1; one build serves both CPU tiers (ncnn +# dispatches its SIMD at run time). +# +# The models are vs-mlrt's ONNX conversions (2.4-5.2 MB each), shipped only +# inside one ~810 MB archive per release: fetched pinned and hash-checked, the +# five VapourBox uses extracted and each checked by its own hash. Keep every pin +# and hash in step with download-deps-{macos.sh,windows.ps1} and +# gpu_plugin::UPSCALE_MODELS. +VSMLRT_TAG="v15.16" +VSMLRT_MODELS_SHA256="d4eaa8c1c4459082185a9e1a2c54d7e2c776abeea6aef6aee0cb8f6c151dfb02" +PROTOBUF_TAG="v3.21.12" +ONNX_COMMIT="b86cc54efce19530fb953e4b21f57e6b3888534c" +NCNN_TAG="20260526" +NCNN_UBUNTU_SHA256="c6daf0c8fb2546de420c7b320e2ce62ba764163ace0ba63a2ea0cdfca70d684d" +UPSCALE_MODEL_HASHES=( + "RealESRGANv2/RealESRGANv2-animevideo-xsx2.onnx 6fad590f96df553a5cc348a9c5129e7874504a7dcd414e4a15fb0b281fca7e0c" + "RealESRGANv2/RealESRGANv2-animevideo-xsx4.onnx 414de3a5da4c56652c99f3bc0b960910e3561835a6c7a9cca546475bc4792158" + "cugan/up2x-latest-no-denoise.onnx 3587278c3a7e62bbcb00a23b4de291ff906f62a88856ee457cbe4211cc593574" + "cugan/up2x-latest-conservative.onnx 555bca3eb90caa37c7e1a02aec9cda3ca8479f252746c1885d382f87312f651c" + "cugan/up2x-latest-denoise3x.onnx 7bcb0510f9f989e681a51d39b7170883d3d5e75f1ec23ef5146e8ea50c3cc845" +) +# VSNCNN is an API 3 plugin (VapourSynth.h / VSHelper.h). R78 still loads +# those, but the build's own include dir need not carry the old headers, so +# they are fetched from the same VapourSynth tag, by hash. +VSAPI3_TAG="R78" +VSAPI3_HEADER_HASHES=( + "VapourSynth.h 0983c937cd369b7e1766913edd3c859ea1f93a79671579729f24eb90a91ce910" + "VSHelper.h 11e2c4977127632aa0fb84c67d5d7eb7244764c52a94500797d4a3cac013e382" +) +NCNN_DIR="$DEPS_DIR/vapoursynth/ncnn" +echo "" +echo "=== Installing VSNCNN ($VSMLRT_TAG) + upscaling models ===" +if [ "$ARCH" != "x86_64" ]; then + echo " Skipped on $ARCH: Tencent publishes no aarch64 Linux ncnn" +elif [ "$FORCE" = false ] && [ -f "$NCNN_DIR/libvsncnn.so" ] \ + && [ -f "$NCNN_DIR/models/cugan/up2x-latest-denoise3x.onnx" ]; then + echo " VSNCNN already exists, skipping" +else + rm -rf "$NCNN_DIR" + if ( + set -e + W="$BUILD_DIR/vsncnn" + rm -rf "$W" && mkdir -p "$W" "$NCNN_DIR/models" + cd "$W" + curl -fsSL "https://github.com/protocolbuffers/protobuf/archive/refs/tags/$PROTOBUF_TAG.tar.gz" | tar xz + cmake -S "protobuf-${PROTOBUF_TAG#v}/cmake" -B pb-build -G Ninja -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_BUILD_SHARED_LIBS=OFF -Dprotobuf_BUILD_TESTS=OFF \ + -DCMAKE_POLICY_VERSION_MINIMUM=3.5 >/dev/null + cmake --build pb-build >/dev/null + cmake --install pb-build --prefix "$W/pb" >/dev/null + mkdir onnx && git -C onnx init -q && git -C onnx remote add origin https://github.com/onnx/onnx.git + git -C onnx fetch -q --depth 1 origin "$ONNX_COMMIT" && git -C onnx checkout -q FETCH_HEAD + cmake -S onnx -B onnx-build -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ + -DCMAKE_PREFIX_PATH="$W/pb" -DProtobuf_PROTOC_EXECUTABLE="$W/pb/bin/protoc" \ + -DONNX_USE_LITE_PROTO=ON -DONNX_USE_PROTOBUF_SHARED_LIBS=OFF -DONNX_GEN_PB_TYPE_STUBS=OFF \ + -DONNX_ML=0 -DONNX_BUILD_PYTHON=OFF -DCMAKE_POLICY_VERSION_MINIMUM=3.5 >/dev/null + cmake --build onnx-build >/dev/null + cmake --install onnx-build --prefix "$W/onnx-inst" >/dev/null + curl -fsSL -o ncnn.zip \ + "https://github.com/Tencent/ncnn/releases/download/$NCNN_TAG/ncnn-$NCNN_TAG-ubuntu-2404.zip" + echo "$NCNN_UBUNTU_SHA256 ncnn.zip" | sha256sum -c - + unzip -q ncnn.zip + mkdir -p "$W/vsapi3" + for entry in "${VSAPI3_HEADER_HASHES[@]}"; do + h="${entry%% *}"; sum="${entry##* }" + curl -fsSL -o "$W/vsapi3/$h" \ + "https://raw.githubusercontent.com/vapoursynth/vapoursynth/$VSAPI3_TAG/include/$h" + echo "$sum $W/vsapi3/$h" | sha256sum -c - + done + git clone -q --depth 1 --branch "$VSMLRT_TAG" https://github.com/AmusementClub/vs-mlrt.git vs-mlrt + cmake -S vs-mlrt/vsncnn -B vsncnn-build -G Ninja -DCMAKE_BUILD_TYPE=Release \ + -DVAPOURSYNTH_INCLUDE_DIRECTORY="$W/vsapi3" \ + -Dprotobuf_DIR="$W/pb/lib/cmake/protobuf" -DONNX_DIR="$W/onnx-inst/lib/cmake/ONNX" \ + -Dncnn_DIR="$W/ncnn-$NCNN_TAG-ubuntu-2404/lib/cmake/ncnn" -DCMAKE_CXX_STANDARD=20 >/dev/null + cmake --build vsncnn-build >/dev/null + cp vsncnn-build/libvsncnn.so "$NCNN_DIR/libvsncnn.so" + chmod 755 "$NCNN_DIR/libvsncnn.so" + curl -fsSL -o models.7z \ + "https://github.com/AmusementClub/vs-mlrt/releases/download/$VSMLRT_TAG/models.$VSMLRT_TAG.7z" + echo "$VSMLRT_MODELS_SHA256 models.7z" | sha256sum -c - + wanted=() + for entry in "${UPSCALE_MODEL_HASHES[@]}"; do wanted+=("models/${entry%% *}"); done + # One pass: the archive is solid, so each separate extraction would + # decompress it from the start again. + if command -v 7z >/dev/null; then + 7z x -y -bd models.7z "${wanted[@]}" >/dev/null + else + bsdtar -xf models.7z "${wanted[@]}" + fi + for entry in "${UPSCALE_MODEL_HASHES[@]}"; do + rel="${entry%% *}"; sum="${entry##* }" + echo "$sum models/$rel" | sha256sum -c - + mkdir -p "$NCNN_DIR/models/$(dirname "$rel")" + mv "models/$rel" "$NCNN_DIR/models/$rel" + done + cd "$BUILD_DIR" && rm -rf "$W" + ); then + echo " Installed VSNCNN -> vapoursynth/ncnn/" + BUILT_PLUGINS+=("vsncnn") + else + rm -rf "$NCNN_DIR" + echo " Failed to install VSNCNN" + FAILED_PLUGINS+=("vsncnn") + fi +fi + # ============================================================================ # Download NNEDI3 weights # ============================================================================ diff --git a/Scripts/download-deps-macos.sh b/Scripts/download-deps-macos.sh index cb7fbdf1..788b21d0 100755 --- a/Scripts/download-deps-macos.sh +++ b/Scripts/download-deps-macos.sh @@ -340,6 +340,10 @@ echo "" echo "=== Building VapourSynth from source ===" VS_TAG="R78" +# BM3D is built from source on both arches (x86_64 in its own block, arm64 via +# build_plugin). Keep in step with download-deps-linux.sh and the wheel version +# in download-deps-windows.ps1 (r12 == wheel 12.0). +BM3D_TAG="r12" VS_BUILD_DIR="$BUILD_DIR/vapoursynth" VS_INSTALL_DIR="$BUILD_DIR/vapoursynth-install" @@ -808,6 +812,7 @@ build_plugin() { local repo="$2" local output_lib="$3" local build_cmd="$4" + local tag="${5:-}" # optional git tag; empty builds the default branch if [ "$FORCE" = false ] && [ -f "$PLUGINS_DIR/$output_lib" ]; then echo " $name already exists, skipping" @@ -818,7 +823,7 @@ build_plugin() { echo "=== Building $name ===" rm -rf "$name" - if ! git clone --depth 1 "$repo" "$name" 2>/dev/null; then + if ! git clone --depth 1 ${tag:+--branch "$tag"} "$repo" "$name" 2>/dev/null; then echo " Failed to clone $name" FAILED_PLUGINS+=("$name") return 1 @@ -1169,6 +1174,43 @@ PYEOF cd "$BUILD_DIR" fi + # BM3D (core.bm3d - Noise Reduction's BM3D method). Stefan-Olt has no + # current x86_64 build and the PyPI wheel is minos 15.0 (our floor is 12.0), + # so build the same tag as arm64 from source. Its meson.build probes the host + # python for the VapourSynth headers; replace that with ours, as for vivtc. + # fftw is repointed at lib/ by the generic pass at the end of this script. + if [ "$FORCE" = true ] || [ ! -f "$PLUGINS_DIR/libbm3d.dylib" ]; then + echo ""; echo "=== Building BM3D $BM3D_TAG (x86_64) ===" + rm -rf bm3d + if git clone --depth 1 --branch "$BM3D_TAG" https://github.com/HomeOfVapourSynthEvolution/VapourSynth-BM3D.git bm3d; then + VS_INC_DIR="$VS_INC_DIR" python3 - bm3d/meson.build <<'PYEOF' +import os, re, sys +path = sys.argv[1] +inc = os.environ["VS_INC_DIR"] +s = open(path).read() +s = re.sub(r"run_command\(.*?\)\.stdout\(\)\.strip\(\)", + repr(inc), s, flags=re.S) +open(path, "w").write(s) +PYEOF + if PKG_CONFIG_PATH="$VS_PC_DIR:$BREW_PREFIX/lib/pkgconfig:${PKG_CONFIG_PATH:-}" \ + meson setup bm3d/build bm3d --buildtype=release \ + && ninja -C bm3d/build; then + lib=$(find bm3d/build -name "*.dylib" -type f | head -1) + if [ -n "$lib" ]; then + cp "$lib" "$PLUGINS_DIR/libbm3d.dylib" + install_name_tool -id "@loader_path/libbm3d.dylib" "$PLUGINS_DIR/libbm3d.dylib" 2>/dev/null || true + codesign -s - -f "$PLUGINS_DIR/libbm3d.dylib" 2>/dev/null || true + echo " Built BM3D" + else echo " no dylib"; FAILED_PLUGINS+=("bm3d"); fi + else + echo " Failed to build BM3D"; FAILED_PLUGINS+=("bm3d") + fi + else + echo " Failed to clone BM3D"; FAILED_PLUGINS+=("bm3d") + fi + cd "$BUILD_DIR" + fi + # KNLMeansCL (OpenCL denoiser - core.knlm.KNLMeansCL). Used by QTGMC when its # Denoiser is set to "knlmeanscl" (havsfunc). Stefan-Olt doesn't ship an # x86_64 build, so build from source like the arm64 path does. Its meson.build @@ -1408,11 +1450,12 @@ build_plugin "tcanny" \ "libtcanny.dylib" \ "meson setup build --buildtype=release && ninja -C build" -# BM3D +# BM3D (core.bm3d - Noise Reduction's BM3D method; x86_64 builds it above) build_plugin "bm3d" \ "https://github.com/HomeOfVapourSynthEvolution/VapourSynth-BM3D.git" \ "libbm3d.dylib" \ - "meson setup build --buildtype=release && ninja -C build" + "meson setup build --buildtype=release && ninja -C build" \ + "$BM3D_TAG" # KNLMeansCL (OpenCL denoiser) build_plugin "knlmeanscl" \ @@ -2092,22 +2135,113 @@ else fi fi # end vszip install +# ============================================================================ +# descale — the Descale pass (core.descale.De*) +# ============================================================================ +# The PyPI wheels, pinned by hash: macOS arm64 is minos 11.0 and x86_64 10.13, +# both under their bundle's floor. Its AVX2 kernel is a separate object chosen +# at run time (opt=0), so nothing AVX runs at load and one file serves both x64 +# tiers. Keep DESCALE_VERSION in step with download-deps-{linux.sh,windows.ps1}. +DESCALE_VERSION="12" +if [ "$ARCH" = "arm64" ]; then + DESCALE_TAG="macosx_11_0_arm64" + DESCALE_SHA256="101f146b32a3d5f7439f4aba663576fb74ab146b29cce520bb1d0b240bb1b2d2" +else + DESCALE_TAG="macosx_10_13_x86_64" + DESCALE_SHA256="9206419849d74db3bac8fb5821f3feb5286450ad2df5f31776ba1e3ba6140347" +fi +DESCALE_OUT="$PLUGINS_DIR/libdescale.dylib" +echo "" +echo "=== Installing descale $DESCALE_VERSION ===" +if [ "$FORCE" = false ] && [ -f "$DESCALE_OUT" ]; then + echo " descale already exists, skipping" +else + rm -f "$DESCALE_OUT" + if ( + set -e + cd "$BUILD_DIR" + rm -rf descale-whl descale.whl + url=$("$PYTHON_BIN" - "$DESCALE_VERSION" "$DESCALE_TAG" <<'PYEOF2' +import json, sys, urllib.request +ver, tag = sys.argv[1], sys.argv[2] +d = json.load(urllib.request.urlopen(f"https://pypi.org/pypi/vapoursynth-descale/{ver}/json")) +print(next(f["url"] for f in d["urls"] if f["filename"].endswith(tag + ".whl"))) +PYEOF2 +) + curl -fsSL -o descale.whl "$url" + echo "$DESCALE_SHA256 descale.whl" | shasum -a 256 -c - + mkdir descale-whl && unzip -q descale.whl -d descale-whl + cp descale-whl/vapoursynth/plugins/libdescale.dylib "$DESCALE_OUT" + chmod u+w "$DESCALE_OUT" + install_name_tool -id "@loader_path/libdescale.dylib" "$DESCALE_OUT" + codesign -s - -f "$DESCALE_OUT" + rm -rf descale-whl descale.whl + ); then + echo " Installed descale -> libdescale.dylib" + BUILT_PLUGINS+=("descale") + else + rm -f "$DESCALE_OUT" + echo " Failed to install descale" + FAILED_PLUGINS+=("descale") + fi +fi + +# ============================================================================ +# MoltenVK — the Vulkan implementation both ncnn plugins (RIFE, VSNCNN) call +# ============================================================================ +# Neither plugin bundles one: each leaves vkGetInstanceProcAddr for whatever is +# already in the process and segfaults (or fails to find a GPU) without it. So +# MoltenVK ships in vapoursynth/moltenvk/ (arm64 slice only, the only arch with +# either plugin) and both templates preload it with RTLD_GLOBAL. +MOLTENVK_VERSION="1.4.2" +MOLTENVK_SHA256="f95765a6229cb7b915990a2890ce12ebe36a730b021545d3d52ae69ce4c4024e" +MOLTENVK_DIR="$DEPS_DIR/vapoursynth/moltenvk" +echo "" +echo "=== Installing MoltenVK $MOLTENVK_VERSION ===" +if [ "$ARCH" != "arm64" ]; then + echo " Skipped on $ARCH: no ncnn plugin ships here (see RIFE and VSNCNN below)" +elif [ "$FORCE" = false ] && [ -f "$MOLTENVK_DIR/libMoltenVK.dylib" ]; then + echo " MoltenVK already exists, skipping" +else + rm -rf "$MOLTENVK_DIR" + if ( + set -e + mkdir -p "$MOLTENVK_DIR" + cd "$BUILD_DIR" + rm -rf moltenvk MoltenVK-macos.tar + curl -fsSL -o MoltenVK-macos.tar \ + "https://github.com/KhronosGroup/MoltenVK/releases/download/v$MOLTENVK_VERSION/MoltenVK-macos.tar" + echo "$MOLTENVK_SHA256 MoltenVK-macos.tar" | shasum -a 256 -c - + mkdir moltenvk + tar -xf MoltenVK-macos.tar -C moltenvk + lipo moltenvk/MoltenVK/MoltenVK/dynamic/dylib/macOS/libMoltenVK.dylib \ + -thin arm64 -output "$MOLTENVK_DIR/libMoltenVK.dylib" + cp moltenvk/MoltenVK/LICENSE "$MOLTENVK_DIR/MoltenVK-LICENSE.txt" + rm -rf moltenvk MoltenVK-macos.tar + chmod u+w "$MOLTENVK_DIR/libMoltenVK.dylib" + codesign -s - -f "$MOLTENVK_DIR/libMoltenVK.dylib" + ); then + echo " Installed MoltenVK -> vapoursynth/moltenvk/" + BUILT_PLUGINS+=("moltenvk") + else + rm -rf "$MOLTENVK_DIR" + echo " Failed to install MoltenVK" + FAILED_PLUGINS+=("moltenvk") + fi +fi + # ============================================================================ # RIFE 4.26 — the Frame Rate pass's neural interpolator (ncnn on Vulkan) # ============================================================================ # Lives in vapoursynth/rife/, NOT the autoload directory: the template loads it -# only for a RIFE job (see worker/src/rife.rs). Keep RIFE_TAG, the model commit -# and the model hashes in step with download-deps-{linux.sh,windows.ps1}. +# only for a RIFE job (see worker/src/gpu_plugin.rs). Keep RIFE_TAG, the model +# commit and the model hashes in step with download-deps-{linux.sh,windows.ps1}. # # arm64 only. The upstream x86_64 dylib targets macOS 13 against this arch's # 12.0 floor (issue #39) and runs AVX in a static initializer # (check-load-time-simd.py fails it), so Intel Macs need a source build first; -# the app reports RIFE as unavailable there. -# -# Neither dylib bundles a Vulkan implementation: it leaves vkGetInstanceProcAddr -# for whatever is already in the process, and segfaults without one. So -# MoltenVK ships beside it (arm64 slice only) and the script preloads it with -# RTLD_GLOBAL. +# the app reports RIFE as unavailable there. It resolves Vulkan from the +# MoltenVK installed above. # # The model is the ncnn conversion committed to the plugin's repo (there is no # release asset for it), fetched at a pinned commit and checked by hash. @@ -2117,15 +2251,12 @@ RIFE_MODEL_COMMIT="96fdcecdf982ef3237e16e2b105eea849fa022ab" RIFE_MODEL_NAME="rife-v4.26_ensembleFalse" RIFE_MODEL_BIN_SHA256="94d58e30b75d7c7609cfa6f3bdad524deddd14f5f75e85c36d2f827ef5c64731" RIFE_MODEL_PARAM_SHA256="79f16c28903f93f8308f0c4c947f8c7e0c17d99a57b85473e5f298dd578d137b" -MOLTENVK_VERSION="1.4.2" -MOLTENVK_SHA256="f95765a6229cb7b915990a2890ce12ebe36a730b021545d3d52ae69ce4c4024e" RIFE_DIR="$DEPS_DIR/vapoursynth/rife" echo "" echo "=== Installing RIFE ($RIFE_TAG, model $RIFE_MODEL_NAME) ===" if [ "$ARCH" != "arm64" ]; then echo " Skipped on $ARCH: no compatible prebuilt (see above)" elif [ "$FORCE" = false ] && [ -f "$RIFE_DIR/librife.dylib" ] \ - && [ -f "$RIFE_DIR/libMoltenVK.dylib" ] \ && [ -f "$RIFE_DIR/models/$RIFE_MODEL_NAME/flownet.bin" ]; then echo " RIFE already exists, skipping" else @@ -2145,19 +2276,8 @@ else echo "$RIFE_MODEL_BIN_SHA256 flownet.bin" | shasum -a 256 -c - echo "$RIFE_MODEL_PARAM_SHA256 flownet.param" | shasum -a 256 -c - mv flownet.bin flownet.param "$RIFE_DIR/models/$RIFE_MODEL_NAME/" - rm -rf moltenvk MoltenVK-macos.tar - curl -fsSL -o MoltenVK-macos.tar \ - "https://github.com/KhronosGroup/MoltenVK/releases/download/v$MOLTENVK_VERSION/MoltenVK-macos.tar" - echo "$MOLTENVK_SHA256 MoltenVK-macos.tar" | shasum -a 256 -c - - mkdir moltenvk - tar -xf MoltenVK-macos.tar -C moltenvk - lipo moltenvk/MoltenVK/MoltenVK/dynamic/dylib/macOS/libMoltenVK.dylib \ - -thin arm64 -output "$RIFE_DIR/libMoltenVK.dylib" - cp moltenvk/MoltenVK/LICENSE "$RIFE_DIR/MoltenVK-LICENSE.txt" - rm -rf moltenvk MoltenVK-macos.tar - chmod u+w "$RIFE_DIR"/*.dylib + chmod u+w "$RIFE_DIR/librife.dylib" codesign -s - -f "$RIFE_DIR/librife.dylib" - codesign -s - -f "$RIFE_DIR/libMoltenVK.dylib" ); then echo " Installed RIFE -> vapoursynth/rife/" BUILT_PLUGINS+=("rife") @@ -2168,6 +2288,132 @@ else fi fi +# ============================================================================ +# VSNCNN (vs-mlrt) + Real-ESRGAN / Real-CUGAN — Crop & Resize's AI upscaling +# ============================================================================ +# vs-mlrt publishes VSNCNN for Windows only, so it is built here from the +# pinned tag the way upstream's linux-ncnn.yml builds it: static protobuf and +# onnx from source, ncnn from Tencent's prebuilt macOS Vulkan frameworks. The +# frameworks are compiled straight in (they carry no CMake package config), and +# vkGetInstanceProcAddr is left to the MoltenVK preloaded above +# (-undefined dynamic_lookup), exactly as for RIFE. Lives in vapoursynth/ncnn/, +# never autoloaded (see worker/src/gpu_plugin.rs). arm64 only, as for RIFE. +# +# The models are vs-mlrt's ONNX conversions, 2.4-5.2 MB each. vs-mlrt ships +# them only inside one ~810 MB archive per release, so that is fetched (pinned +# and hash-checked), the five files VapourBox uses are extracted, and each is +# checked by its own hash. Keep every pin and hash in step with +# download-deps-{linux.sh,windows.ps1} and gpu_plugin::UPSCALE_MODELS. +VSMLRT_TAG="v15.16" +VSMLRT_MODELS_SHA256="d4eaa8c1c4459082185a9e1a2c54d7e2c776abeea6aef6aee0cb8f6c151dfb02" +PROTOBUF_TAG="v3.21.12" +ONNX_COMMIT="b86cc54efce19530fb953e4b21f57e6b3888534c" +NCNN_TAG="20260526" +NCNN_MACOS_VULKAN_SHA256="e3ec1d473390a52c288968ecf1321fd2fdd25d8eaa2031a6b7d5c97f3e6961b4" +UPSCALE_MODEL_HASHES=( + "RealESRGANv2/RealESRGANv2-animevideo-xsx2.onnx 6fad590f96df553a5cc348a9c5129e7874504a7dcd414e4a15fb0b281fca7e0c" + "RealESRGANv2/RealESRGANv2-animevideo-xsx4.onnx 414de3a5da4c56652c99f3bc0b960910e3561835a6c7a9cca546475bc4792158" + "cugan/up2x-latest-no-denoise.onnx 3587278c3a7e62bbcb00a23b4de291ff906f62a88856ee457cbe4211cc593574" + "cugan/up2x-latest-conservative.onnx 555bca3eb90caa37c7e1a02aec9cda3ca8479f252746c1885d382f87312f651c" + "cugan/up2x-latest-denoise3x.onnx 7bcb0510f9f989e681a51d39b7170883d3d5e75f1ec23ef5146e8ea50c3cc845" +) +# VSNCNN is an API 3 plugin (VapourSynth.h / VSHelper.h). R78 still loads +# those, but the build's own include dir need not carry the old headers, so +# they are fetched from the same VapourSynth tag, by hash. +VSAPI3_TAG="R78" +VSAPI3_HEADER_HASHES=( + "VapourSynth.h 0983c937cd369b7e1766913edd3c859ea1f93a79671579729f24eb90a91ce910" + "VSHelper.h 11e2c4977127632aa0fb84c67d5d7eb7244764c52a94500797d4a3cac013e382" +) +NCNN_DIR="$DEPS_DIR/vapoursynth/ncnn" +echo "" +echo "=== Installing VSNCNN ($VSMLRT_TAG) + upscaling models ===" +if [ "$ARCH" != "arm64" ]; then + echo " Skipped on $ARCH: no MoltenVK here (see above)" +elif [ "$FORCE" = false ] && [ -f "$NCNN_DIR/libvsncnn.dylib" ] \ + && [ -f "$NCNN_DIR/models/cugan/up2x-latest-denoise3x.onnx" ]; then + echo " VSNCNN already exists, skipping" +else + rm -rf "$NCNN_DIR" + if ( + set -e + W="$BUILD_DIR/vsncnn" + rm -rf "$W" && mkdir -p "$W" "$NCNN_DIR/models" + cd "$W" + export MACOSX_DEPLOYMENT_TARGET="$MACOS_MIN_VERSION" + # protobuf, static. + curl -fsSL "https://github.com/protocolbuffers/protobuf/archive/refs/tags/$PROTOBUF_TAG.tar.gz" | tar xz + cmake -S "protobuf-${PROTOBUF_TAG#v}/cmake" -B pb-build -G Ninja -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_BUILD_SHARED_LIBS=OFF -Dprotobuf_BUILD_TESTS=OFF \ + -DCMAKE_OSX_ARCHITECTURES=arm64 -DCMAKE_POLICY_VERSION_MINIMUM=3.5 >/dev/null + cmake --build pb-build >/dev/null + cmake --install pb-build --prefix "$W/pb" >/dev/null + # onnx, static, lite protobuf, at upstream's pinned commit. + mkdir onnx && git -C onnx init -q && git -C onnx remote add origin https://github.com/onnx/onnx.git + git -C onnx fetch -q --depth 1 origin "$ONNX_COMMIT" && git -C onnx checkout -q FETCH_HEAD + cmake -S onnx -B onnx-build -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ + -DCMAKE_PREFIX_PATH="$W/pb" -DProtobuf_PROTOC_EXECUTABLE="$W/pb/bin/protoc" \ + -DONNX_USE_LITE_PROTO=ON -DONNX_USE_PROTOBUF_SHARED_LIBS=OFF -DONNX_GEN_PB_TYPE_STUBS=OFF \ + -DONNX_ML=0 -DONNX_BUILD_PYTHON=OFF -DCMAKE_OSX_ARCHITECTURES=arm64 \ + -DCMAKE_POLICY_VERSION_MINIMUM=3.5 >/dev/null + cmake --build onnx-build >/dev/null + cmake --install onnx-build --prefix "$W/onnx-inst" >/dev/null + # ncnn, prebuilt (universal static frameworks). + curl -fsSL -o ncnn.zip \ + "https://github.com/Tencent/ncnn/releases/download/$NCNN_TAG/ncnn-$NCNN_TAG-macos-vulkan.zip" + echo "$NCNN_MACOS_VULKAN_SHA256 ncnn.zip" | shasum -a 256 -c - + mkdir ncnn && unzip -q ncnn.zip -d ncnn + N="$W/ncnn" + mkdir -p "$W/vsapi3" + for entry in "${VSAPI3_HEADER_HASHES[@]}"; do + h="${entry%% *}"; sum="${entry##* }" + curl -fsSL -o "$W/vsapi3/$h" \ + "https://raw.githubusercontent.com/vapoursynth/vapoursynth/$VSAPI3_TAG/include/$h" + echo "$sum $W/vsapi3/$h" | shasum -a 256 -c - + done + # VSNCNN itself. + git clone -q --depth 1 --branch "$VSMLRT_TAG" https://github.com/AmusementClub/vs-mlrt.git vs-mlrt + echo "#define VERSION \"$VSMLRT_TAG\"" > vs-mlrt/vsncnn/config.h + clang++ -std=c++20 -O2 -fPIC -shared -arch arm64 -mmacosx-version-min="$MACOS_MIN_VERSION" \ + -fvisibility=hidden \ + vs-mlrt/vsncnn/vs_ncnn.cpp vs-mlrt/vsncnn/onnx2ncnn.cpp vs-mlrt/common/onnx_utils.cpp \ + -Ivs-mlrt/vsncnn -I"$W/vsapi3" -I"$VS_INC_DIR" -I"$N/ncnn.framework/Headers" -I"$N/ncnn.framework/Headers/ncnn" \ + -I"$N/openmp.framework/Headers" -I"$W/onnx-inst/include" -I"$W/pb/include" \ + -DONNX_NAMESPACE=onnx -DONNX_USE_LITE_PROTO=1 \ + "$N/ncnn.framework/Versions/A/ncnn" "$N/glslang.framework/Versions/A/glslang" \ + "$N/openmp.framework/Versions/A/openmp" \ + "$W/onnx-inst/lib/libonnx.a" "$W/onnx-inst/lib/libonnx_proto.a" "$W/pb/lib/libprotobuf-lite.a" \ + -framework Foundation -framework Metal -framework QuartzCore -framework IOSurface -framework IOKit \ + -Wl,-undefined,dynamic_lookup -Wl,-install_name,@loader_path/libvsncnn.dylib \ + -o "$NCNN_DIR/libvsncnn.dylib" + codesign -s - -f "$NCNN_DIR/libvsncnn.dylib" + # Models. + curl -fsSL -o models.7z \ + "https://github.com/AmusementClub/vs-mlrt/releases/download/$VSMLRT_TAG/models.$VSMLRT_TAG.7z" + echo "$VSMLRT_MODELS_SHA256 models.7z" | shasum -a 256 -c - + # One pass: the archive is solid, so each separate extraction would + # decompress it from the start again. + wanted=() + for entry in "${UPSCALE_MODEL_HASHES[@]}"; do wanted+=("models/${entry%% *}"); done + tar -xf models.7z "${wanted[@]}" + for entry in "${UPSCALE_MODEL_HASHES[@]}"; do + rel="${entry%% *}"; sum="${entry##* }" + echo "$sum models/$rel" | shasum -a 256 -c - + mkdir -p "$NCNN_DIR/models/$(dirname "$rel")" + mv "models/$rel" "$NCNN_DIR/models/$rel" + done + rm -f models.7z + cd "$BUILD_DIR" && rm -rf "$W" + ); then + echo " Installed VSNCNN -> vapoursynth/ncnn/" + BUILT_PLUGINS+=("vsncnn") + else + rm -rf "$NCNN_DIR" + echo " Failed to install VSNCNN" + FAILED_PLUGINS+=("vsncnn") + fi +fi + # ============================================================================ # Download NNEDI3 weights # ============================================================================ diff --git a/Scripts/download-deps-windows.ps1 b/Scripts/download-deps-windows.ps1 index ca0bbc96..a6567c65 100644 --- a/Scripts/download-deps-windows.ps1 +++ b/Scripts/download-deps-windows.ps1 @@ -774,7 +774,7 @@ if (-not (Test-Path "$PluginsDir\libakarin.dll")) { } # ============================================================================= -# Bwdif + DeDot + vszip - PyPI wheels +# Bwdif + DeDot + vszip + BM3D + descale - PyPI wheels # ============================================================================= # None of these upstreams publishes a Windows release asset, so each comes from # its PyPI wheel, resolved through the JSON API the same way akarin is (never @@ -801,7 +801,14 @@ if (-not (Test-Path "$PluginsDir\libakarin.dll")) { $WheelPlugins = @( @{ Name = "bwdif"; Package = "vapoursynth-bwdif"; Version = "5.1"; Dll = "bwdif.dll" }, @{ Name = "dedot"; Package = "vapoursynth-dedot"; Version = "3.0"; Dll = "dedot.dll" }, - @{ Name = "vszip"; Package = "vapoursynth-vszip"; Version = "22.1.0"; Dll = "vszip.dll"; Src = "vszip\vszip.dll" } + @{ Name = "vszip"; Package = "vapoursynth-vszip"; Version = "22.1.0"; Dll = "vszip.dll"; Src = "vszip\vszip.dll" }, + # core.bm3d (Noise Reduction's BM3D method). Wheel 12.0 is git tag r12, as + # built from source by download-deps-{macos,linux}.sh. fftw is linked in + # statically: the DLL imports only the UCRT. + @{ Name = "bm3d"; Package = "vapoursynth-bm3d"; Version = "12.0"; Dll = "bm3d.dll" }, + # core.descale (the Descale pass). Its AVX2 kernel is chosen at run time, so + # one DLL serves both tiers. Imports VCRUNTIME140, which Python ships. + @{ Name = "descale"; Package = "vapoursynth-descale"; Version = "12"; Dll = "libdescale.dll" } ) foreach ($W in $WheelPlugins) { @@ -845,7 +852,7 @@ foreach ($W in $WheelPlugins) { # RIFE 4.26 - the Frame Rate pass's neural interpolator (ncnn on Vulkan) # ============================================================================= # Lives in vapoursynth\rife\, NOT vs-plugins: the template loads it only for a -# RIFE job (see worker\src\rife.rs), so a machine without a Vulkan driver, or +# RIFE job (see worker\src\gpu_plugin.rs), so a machine without a Vulkan driver, or # without the MSVC OpenMP runtime (VCOMP140.DLL) the DLL imports, fails that # method alone - and the app's probe names which one is missing. vulkan-1.dll # comes with the GPU driver and is not ours to ship. Keep $RifeTag, the model @@ -891,6 +898,81 @@ if (-not ((Test-Path "$RifeDir\librife.dll") -and (Test-Path "$RifeModelDir\flow Write-Host " RIFE already installed" -ForegroundColor Gray } +# ============================================================================= +# VSNCNN (vs-mlrt) + Real-ESRGAN / Real-CUGAN - Crop & Resize's AI upscaling +# ============================================================================= +# vs-mlrt's own Windows build, pinned and hash-checked. Lives in +# vapoursynth\ncnn\, never in vs-plugins (see worker\src\gpu_plugin.rs). It links +# its runtime statically (no VC++ redistributable) and loads vulkan-1.dll, the +# GPU driver's, at run time. One file serves both CPU tiers: ncnn dispatches +# its SIMD at run time. +# +# The models are vs-mlrt's ONNX conversions (2.4-5.2 MB each), shipped only +# inside one ~810 MB archive per release: fetched pinned and hash-checked, the +# five VapourBox uses extracted and each checked by its own hash. Keep every pin +# and hash in step with download-deps-{macos,linux}.sh and +# gpu_plugin::UPSCALE_MODELS. Needs 7-Zip, like the other .7z downloads. +$VsmlrtTag = "v15.16" +$VsncnnArchiveSha256 = "2ccf0ec1f84cd60f4a744c04d2a40d988da32f6970542f4f81669d44e9ea2ff1" +$VsmlrtModelsSha256 = "d4eaa8c1c4459082185a9e1a2c54d7e2c776abeea6aef6aee0cb8f6c151dfb02" +$UpscaleModelHashes = [ordered]@{ + "RealESRGANv2/RealESRGANv2-animevideo-xsx2.onnx" = "6fad590f96df553a5cc348a9c5129e7874504a7dcd414e4a15fb0b281fca7e0c" + "RealESRGANv2/RealESRGANv2-animevideo-xsx4.onnx" = "414de3a5da4c56652c99f3bc0b960910e3561835a6c7a9cca546475bc4792158" + "cugan/up2x-latest-no-denoise.onnx" = "3587278c3a7e62bbcb00a23b4de291ff906f62a88856ee457cbe4211cc593574" + "cugan/up2x-latest-conservative.onnx" = "555bca3eb90caa37c7e1a02aec9cda3ca8479f252746c1885d382f87312f651c" + "cugan/up2x-latest-denoise3x.onnx" = "7bcb0510f9f989e681a51d39b7170883d3d5e75f1ec23ef5146e8ea50c3cc845" +} +$NcnnDir = "$FullTargetDir\vapoursynth\ncnn" + +Write-Host "" +Write-Host "Downloading VSNCNN $VsmlrtTag + upscaling models..." -ForegroundColor Yellow +if (-not $Has7z) { + Write-Host " Skipping (7-Zip not installed)" -ForegroundColor Yellow +} elseif (-not ((Test-Path "$NcnnDir\vsncnn.dll") -and (Test-Path "$NcnnDir\models\cugan\up2x-latest-denoise3x.onnx"))) { + try { + Remove-Item $NcnnDir -Recurse -Force -ErrorAction SilentlyContinue + New-Item -ItemType Directory -Force -Path "$NcnnDir\models" | Out-Null + $Work = Join-Path $TempDir "vsncnn" + Remove-Item $Work -Recurse -Force -ErrorAction SilentlyContinue + New-Item -ItemType Directory -Force -Path $Work | Out-Null + + $Archive = Join-Path $Work "vsncnn.7z" + Download-File -Url "https://github.com/AmusementClub/vs-mlrt/releases/download/$VsmlrtTag/VSNCNN-Windows-x64.$VsmlrtTag.7z" -OutFile $Archive + $Got = (Get-FileHash -Path $Archive -Algorithm SHA256).Hash.ToLower() + if ($Got -ne $VsncnnArchiveSha256) { throw "VSNCNN archive sha256 $Got, expected $VsncnnArchiveSha256" } + & $7zPath x $Archive -o"$Work" -y | Out-Null + if ($LASTEXITCODE -ne 0) { throw "7-Zip failed on the VSNCNN archive (exit $LASTEXITCODE)" } + $Dll = Get-ChildItem -Path $Work -Recurse -Filter "vsncnn.dll" | Select-Object -First 1 + if (-not $Dll) { throw "vsncnn.dll missing from the VSNCNN archive" } + Copy-Item $Dll.FullName "$NcnnDir\vsncnn.dll" -Force + + $Models = Join-Path $Work "models.7z" + Download-File -Url "https://github.com/AmusementClub/vs-mlrt/releases/download/$VsmlrtTag/models.$VsmlrtTag.7z" -OutFile $Models + $Got = (Get-FileHash -Path $Models -Algorithm SHA256).Hash.ToLower() + if ($Got -ne $VsmlrtModelsSha256) { throw "models archive sha256 $Got, expected $VsmlrtModelsSha256" } + # One pass: the archive is solid, so separate extractions would each + # decompress it from the start again. + $Wanted = $UpscaleModelHashes.Keys | ForEach-Object { "models/$_" } + & $7zPath x $Models -o"$Work" -y @Wanted | Out-Null + if ($LASTEXITCODE -ne 0) { throw "7-Zip failed on the models archive (exit $LASTEXITCODE)" } + foreach ($Rel in $UpscaleModelHashes.Keys) { + $Src = Join-Path $Work ("models\" + $Rel.Replace('/', '\')) + $Got = (Get-FileHash -Path $Src -Algorithm SHA256).Hash.ToLower() + if ($Got -ne $UpscaleModelHashes[$Rel]) { throw "$Rel sha256 $Got, expected $($UpscaleModelHashes[$Rel])" } + $Dest = Join-Path "$NcnnDir\models" $Rel.Replace('/', '\') + New-Item -ItemType Directory -Force -Path (Split-Path $Dest) | Out-Null + Move-Item $Src $Dest -Force + } + Remove-Item $Work -Recurse -Force -ErrorAction SilentlyContinue + Write-Host " VSNCNN installed -> vapoursynth\ncnn\" -ForegroundColor Green + } catch { + Remove-Item $NcnnDir -Recurse -Force -ErrorAction SilentlyContinue + Write-Host " Failed: $_" -ForegroundColor Red + } +} else { + Write-Host " VSNCNN already installed" -ForegroundColor Gray +} + # ============================================================================= # 5. Python Packages (havsfunc, mvsfunc, adjust) # ============================================================================= diff --git a/app/assets/filters/core/crop_resize.json b/app/assets/filters/core/crop_resize.json index 026d2dae..2473223c 100644 --- a/app/assets/filters/core/crop_resize.json +++ b/app/assets/filters/core/crop_resize.json @@ -246,17 +246,20 @@ "options": [ "nnedi3Rpow2", "eedi3Rpow2", - "spline36" + "spline36", + "realEsrgan", + "realCugan" ], "optional": true, "ui": { "label": "Upscale Method", - "description": "Edge-directed doubling. NNEDI3 is the usual choice; EEDI3 is slower and follows near-horizontal edges better (it uses NNEDI3 to guide its search); Spline36 is plain resampling with no edge direction", + "description": "Edge-directed doubling. NNEDI3 is the usual choice; EEDI3 is slower and follows near-horizontal edges better (it uses NNEDI3 to guide its search); Spline36 is plain resampling with no edge direction. Real-ESRGAN and Real-CUGAN are neural upscalers trained on anime: they redraw clean line art rather than interpolating it, and run on the graphics card (Vulkan). Real-ESRGAN is sharper and faster; Real-CUGAN is gentler and can remove noise as it goes. Use them after deinterlacing: on a combed picture they sharpen the combing too", "widget": "dropdown", "visibleWhen": { "useIntegerUpscale": true } - } + }, + "sinceAppVersion": "1.5.0" }, "upscaleFactor": { "type": "integer", @@ -274,6 +277,27 @@ } } }, + "cuganNoise": { + "type": "enum", + "default": "conservative", + "options": [ + "none", + "conservative", + "denoise3x" + ], + "sinceAppVersion": "1.5.0", + "ui": { + "label": "Real-CUGAN Denoise", + "description": "How much noise Real-CUGAN removes as it upscales: none, conservative (keeps texture) or strong (denoise3x). At 4x it denoises on the first doubling only", + "widget": "dropdown", + "visibleWhen": { + "useIntegerUpscale": true, + "upscaleMethod": [ + "realCugan" + ] + } + } + }, "bicubicB": { "type": "number", "default": 0.0, @@ -840,11 +864,12 @@ }, { "title": "Upscale", - "description": "Enlarge using an edge-directed interpolator rather than a plain resize. Doubles at a time, so pick the factor that reaches or exceeds your target and set a Resize below to land on it exactly.", + "description": "Enlarge using an edge-directed interpolator or a neural upscaler rather than a plain resize. Doubles at a time, so pick the factor that reaches or exceeds your target and set a Resize below to land on it exactly.", "parameters": [ "useIntegerUpscale", "upscaleMethod", - "upscaleFactor" + "upscaleFactor", + "cuganNoise" ], "expanded": false }, @@ -910,6 +935,22 @@ "upscaleMethod": "spline36" } }, + { + "function": "core.ncnn.Model", + "role": "Real-ESRGAN anime video model, on the GPU (vs-mlrt's VSNCNN)", + "activeWhen": { + "useIntegerUpscale": true, + "upscaleMethod": "realEsrgan" + } + }, + { + "function": "core.ncnn.Model", + "role": "Real-CUGAN, on the GPU (vs-mlrt's VSNCNN)", + "activeWhen": { + "useIntegerUpscale": true, + "upscaleMethod": "realCugan" + } + }, { "function": "core.std.AddBorders", "role": "Pad to Fill's letterbox/pillarbox bars, in the border colour", diff --git a/app/assets/filters/core/descale.json b/app/assets/filters/core/descale.json new file mode 100644 index 00000000..9bf13ca2 --- /dev/null +++ b/app/assets/filters/core/descale.json @@ -0,0 +1,265 @@ +{ + "$schema": "https://vapourbox.app/schemas/filter-v1.json", + "id": "descale", + "version": "1.0.0", + "sinceAppVersion": "1.5.0", + "name": "Descale", + "description": "Undo a studio's upscale, then upscale again cleanly", + "longDescription": "Most modern anime — and plenty of other HD video — is drawn or rendered below its delivery resolution (720p, 810p and 873p are common) and upscaled to 1080p with an ordinary resampling kernel. That upscale is what makes line art look soft and haloed. Given the resolution and kernel the studio used, Descale inverts it exactly, recovering the picture as it was drawn, then upscales it back to the source size with NNEDI3, which draws far cleaner lines.\n\nIt only helps when the settings match the source, and harms the picture when they don't — so use Detect, which tries every height and the common kernels on frames from your video and reports whether one stands out. A source that was never upscaled has no native resolution to find, and Detect says so.\n\nCredits, text and overlays are usually drawn at full resolution on top of the upscaled picture; Protect detail keeps those untouched. Brightness only — colour was stored at a lower resolution to begin with. The output keeps the source's size, so Crop & Resize can still upscale it further.", + "category": "enhancement", + "icon": "photo_size_select_large", + "methods": [ + { + "id": "descale", + "name": "Descale", + "description": "Invert the studio's upscale on brightness, re-upscale with NNEDI3", + "function": "custom", + "parameters": [ + "nativeHeight", + "nativeWidth", + "kernel", + "bicubicB", + "bicubicC", + "lanczosTaps", + "border", + "protectDetail", + "detailThreshold" + ] + } + ], + "parameters": { + "enabled": { + "type": "boolean", + "default": false, + "ui": { + "hidden": true + } + }, + "nativeHeight": { + "type": "integer", + "default": 720, + "min": 240, + "max": 2159, + "step": 1, + "ui": { + "label": "Native Height", + "description": "The height the picture was drawn at, below the source's. Use Detect rather than guessing: a wrong value softens the picture", + "widget": "number" + } + }, + "nativeWidth": { + "type": "integer", + "default": 0, + "min": 0, + "max": 3839, + "step": 1, + "ui": { + "label": "Native Width", + "description": "0 works it out from the source's shape, which is right unless the studio also changed the aspect", + "widget": "number" + } + }, + "kernel": { + "type": "enum", + "default": "bicubic", + "options": [ + "bilinear", + "bicubic", + "lanczos", + "spline16", + "spline36", + "spline64" + ], + "ui": { + "label": "Kernel", + "description": "The resampler the studio upscaled with. Bicubic (Catmull-Rom: b 0, c 0.5) and bilinear are the most common", + "widget": "dropdown" + } + }, + "bicubicB": { + "type": "number", + "default": 0.0, + "min": -1.0, + "max": 1.0, + "step": 0.05, + "ui": { + "label": "Bicubic b", + "description": "With c, picks the bicubic curve: Catmull-Rom is b 0 / c 0.5, Mitchell 1/3 / 1/3, Hermite 0 / 0", + "widget": "slider", + "precision": 3, + "visibleWhen": { + "kernel": [ + "bicubic" + ] + } + } + }, + "bicubicC": { + "type": "number", + "default": 0.5, + "min": 0.0, + "max": 1.0, + "step": 0.05, + "ui": { + "label": "Bicubic c", + "description": "See b", + "widget": "slider", + "precision": 3, + "visibleWhen": { + "kernel": [ + "bicubic" + ] + } + } + }, + "lanczosTaps": { + "type": "integer", + "default": 3, + "min": 1, + "max": 8, + "step": 1, + "ui": { + "label": "Lanczos Taps", + "description": "3 is the usual one", + "widget": "slider", + "visibleWhen": { + "kernel": [ + "lanczos" + ] + } + } + }, + "border": { + "type": "enum", + "default": "mirror", + "options": [ + "mirror", + "zero", + "extend" + ], + "ui": { + "label": "Edge Handling", + "description": "How the studio's resampler treated pixels past the frame edge. Mirror is almost always right; a wrong choice shows only as a faint band at the edges", + "widget": "dropdown" + } + }, + "protectDetail": { + "type": "boolean", + "default": true, + "ui": { + "label": "Protect credits and text", + "description": "Keep the source wherever re-applying the kernel does not reproduce it: detail drawn at full resolution, which descaling would smear", + "widget": "checkbox" + } + }, + "detailThreshold": { + "type": "number", + "default": 4.0, + "min": 0.5, + "max": 32.0, + "step": 0.5, + "ui": { + "label": "Protection Threshold", + "description": "How far (in 8-bit levels) the re-applied kernel may miss before a pixel is treated as full-resolution detail. Lower protects more", + "widget": "slider", + "precision": 1, + "visibleWhen": { + "protectDetail": true + } + } + } + }, + "implementation": [ + { + "function": "core.descale.Debilinear", + "role": "inverts the studio's bilinear upscale", + "activeWhen": { + "kernel": "bilinear" + } + }, + { + "function": "core.descale.Debicubic", + "role": "inverts the studio's bicubic upscale", + "activeWhen": { + "kernel": "bicubic" + } + }, + { + "function": "core.descale.Delanczos", + "role": "inverts the studio's Lanczos upscale", + "activeWhen": { + "kernel": "lanczos" + } + }, + { + "function": "core.descale.Despline16", + "role": "inverts the studio's Spline16 upscale", + "activeWhen": { + "kernel": "spline16" + } + }, + { + "function": "core.descale.Despline36", + "role": "inverts the studio's Spline36 upscale", + "activeWhen": { + "kernel": "spline36" + } + }, + { + "function": "core.descale.Despline64", + "role": "inverts the studio's Spline64 upscale", + "activeWhen": { + "kernel": "spline64" + } + }, + { + "function": "core.znedi3.nnedi3", + "role": "re-upscales the recovered picture (core.nnedi3.nnedi3 on ARM)" + }, + { + "function": "core.std.MaskedMerge", + "role": "keeps the source where it was drawn at full resolution", + "activeWhen": { + "protectDetail": true + } + } + ], + "ui": { + "sections": [ + { + "title": "Native resolution", + "parameters": [ + "nativeHeight" + ], + "expanded": true + }, + { + "title": "Studio kernel", + "parameters": [ + "kernel", + "bicubicB", + "bicubicC", + "lanczosTaps" + ], + "expanded": true + }, + { + "title": "Credits and text", + "parameters": [ + "protectDetail", + "detailThreshold" + ], + "expanded": true + }, + { + "title": "Descale tuning", + "parameters": [ + "nativeWidth", + "border" + ], + "expanded": false, + "advancedOnly": true + } + ] + } +} diff --git a/app/assets/filters/core/noise_reduction.json b/app/assets/filters/core/noise_reduction.json index a994225b..9070ab78 100644 --- a/app/assets/filters/core/noise_reduction.json +++ b/app/assets/filters/core/noise_reduction.json @@ -4,7 +4,7 @@ "version": "1.1.0", "name": "Noise Reduction", "description": "Remove video noise and grain", - "longDescription": "Reduces grain, analog noise and colour speckle while trying to keep real detail. Temporal denoisers average across neighbouring frames and are best for steady grain; spatial ones smooth within a single frame and cope better with motion.\n\nUse it on VHS, Hi8 and other noisy captures, and before sharpening or encoding — clean footage compresses far better at the same bitrate. Too much strength smears motion and flattens texture into plastic.", + "longDescription": "Reduces grain, analog noise and colour speckle while trying to keep real detail. Temporal denoisers average across neighbouring frames and are best for steady grain; spatial ones smooth within a single frame and cope better with motion.\n\nUse it on VHS, Hi8 and other noisy captures, and before sharpening or encoding — clean footage compresses far better at the same bitrate. Too much strength smears motion and flattens texture into plastic.\n\nFor anime and other flat-shaded animation, BM3D (advanced mode) is the usual choice: it cleans flat colour without smearing line art.", "category": "cleanup", "icon": "grain", "order": 2, @@ -174,6 +174,34 @@ "td2PostMix", "td2ChromaMotion" ] + }, + { + "id": "bm3d", + "name": "BM3D", + "description": "Block-matching 3D denoiser: finds similar patches across the frame (and neighbouring frames) and cleans them together. Leaves flat colour areas very clean without smearing line art, which is why it is the usual choice for anime. CPU, moderate speed", + "function": "core.bm3d.VBasic + core.bm3d.VFinal", + "parameters": [ + "bm3dSigma", + "bm3dChromaSigma", + "bm3dRadius", + "bm3dFinal", + "bm3dProfile" + ], + "advancedOnly": true + }, + { + "id": "knlmeans", + "name": "KNLMeansCL (GPU)", + "description": "Non-local means on the graphics card. Similar results to BM3D and much faster, but it needs a GPU with working OpenCL; where there is none, VapourBox says so and the job is refused", + "function": "core.knlm.KNLMeansCL", + "parameters": [ + "knlmH", + "knlmD", + "knlmA", + "knlmS", + "knlmChroma" + ], + "advancedOnly": true } ], "parameters": { @@ -200,11 +228,14 @@ "stpresso", "ctmf", "mclean", - "temporal_degrain2" + "temporal_degrain2", + "bm3d", + "knlmeans" ], "ui": { "hidden": true - } + }, + "sinceAppVersion": "1.5.0" }, "smDegrainTr": { "type": "integer", @@ -1044,6 +1075,187 @@ ] } } + }, + "bm3dSigma": { + "type": "number", + "default": 3.0, + "min": 0.5, + "max": 20.0, + "step": 0.5, + "sinceAppVersion": "1.5.0", + "ui": { + "label": "Strength", + "description": "How much noise to remove from brightness. 2–5 suits most sources; higher starts to flatten fine texture", + "widget": "slider", + "precision": 1, + "visibleWhen": { + "method": [ + "bm3d" + ] + } + } + }, + "bm3dChromaSigma": { + "type": "number", + "default": 0.0, + "min": 0.0, + "max": 20.0, + "step": 0.5, + "sinceAppVersion": "1.5.0", + "ui": { + "label": "Colour Strength", + "description": "Noise removal for colour. 0 leaves colour alone (and is faster); use Chroma Denoise for blotchy colour noise", + "widget": "slider", + "precision": 1, + "visibleWhen": { + "method": [ + "bm3d" + ] + } + } + }, + "bm3dRadius": { + "type": "integer", + "default": 1, + "min": 0, + "max": 3, + "step": 1, + "sinceAppVersion": "1.5.0", + "ui": { + "label": "Temporal Radius", + "description": "Frames each side to search for matching patches. 0 works within each frame only; 1 is the usual choice; each step costs about a third more time", + "widget": "slider", + "visibleWhen": { + "method": [ + "bm3d" + ] + } + } + }, + "bm3dFinal": { + "type": "boolean", + "default": true, + "sinceAppVersion": "1.5.0", + "ui": { + "label": "Refine (second pass)", + "description": "Run BM3D's second, Wiener estimate on top of the first. Cleaner flat areas for about half the speed", + "widget": "checkbox", + "visibleWhen": { + "method": [ + "bm3d" + ] + } + } + }, + "bm3dProfile": { + "type": "enum", + "default": "np", + "options": [ + "fast", + "lc", + "np", + "high", + "vn" + ], + "sinceAppVersion": "1.5.0", + "ui": { + "label": "Profile", + "description": "BM3D's own speed/quality presets: fast, lc (low complexity), np (normal), high, vn (very noisy sources)", + "widget": "dropdown", + "visibleWhen": { + "method": [ + "bm3d" + ] + } + } + }, + "knlmH": { + "type": "number", + "default": 1.2, + "min": 0.1, + "max": 10.0, + "step": 0.1, + "sinceAppVersion": "1.5.0", + "ui": { + "label": "Strength", + "description": "How much noise to remove. 0.8–2 suits most sources", + "widget": "slider", + "precision": 1, + "visibleWhen": { + "method": [ + "knlmeans" + ] + } + } + }, + "knlmD": { + "type": "integer", + "default": 1, + "min": 0, + "max": 4, + "step": 1, + "sinceAppVersion": "1.5.0", + "ui": { + "label": "Temporal Radius", + "description": "Frames each side to compare against. 0 works within each frame only", + "widget": "slider", + "visibleWhen": { + "method": [ + "knlmeans" + ] + } + } + }, + "knlmA": { + "type": "integer", + "default": 2, + "min": 1, + "max": 4, + "step": 1, + "sinceAppVersion": "1.5.0", + "ui": { + "label": "Search Radius", + "description": "How far around each pixel to look for similar patches. Larger finds more matches and is much slower", + "widget": "slider", + "visibleWhen": { + "method": [ + "knlmeans" + ] + } + } + }, + "knlmS": { + "type": "integer", + "default": 4, + "min": 0, + "max": 4, + "step": 1, + "sinceAppVersion": "1.5.0", + "ui": { + "label": "Patch Radius", + "description": "Size of the patch compared around each pixel", + "widget": "slider", + "visibleWhen": { + "method": [ + "knlmeans" + ] + } + } + }, + "knlmChroma": { + "type": "boolean", + "default": true, + "sinceAppVersion": "1.5.0", + "ui": { + "label": "Denoise colour too", + "description": "Leave on unless colour is already clean", + "widget": "checkbox", + "visibleWhen": { + "method": [ + "knlmeans" + ] + } + } } }, "ui": { @@ -1186,6 +1398,28 @@ ], "expanded": true, "advancedOnly": true + }, + { + "title": "BM3D", + "parameters": [ + "bm3dSigma", + "bm3dChromaSigma", + "bm3dRadius", + "bm3dFinal", + "bm3dProfile" + ], + "expanded": true + }, + { + "title": "KNLMeansCL", + "parameters": [ + "knlmH", + "knlmD", + "knlmA", + "knlmS", + "knlmChroma" + ], + "expanded": true } ] }, diff --git a/app/assets/filters/manifest.json b/app/assets/filters/manifest.json index c37d0a80..00ac3319 100644 --- a/app/assets/filters/manifest.json +++ b/app/assets/filters/manifest.json @@ -19,5 +19,6 @@ "frame_rate.json", "deflicker.json", "edge_repair.json", - "ghost_removal.json" + "ghost_removal.json", + "descale.json" ] diff --git a/app/lib/models/crop_resize_parameters.dart b/app/lib/models/crop_resize_parameters.dart index 8cf2bb57..65d706d1 100644 --- a/app/lib/models/crop_resize_parameters.dart +++ b/app/lib/models/crop_resize_parameters.dart @@ -32,6 +32,12 @@ enum UpscaleMethod { eedi3Rpow2, @JsonValue('spline36') spline36, + /// Real-ESRGAN's anime video model, on the GPU (needs Vulkan). + @JsonValue('realEsrgan') + realEsrgan, + /// Real-CUGAN, on the GPU (needs Vulkan). + @JsonValue('realCugan') + realCugan, } /// What to do with the pixel aspect ratio (SAR) on output. @@ -177,6 +183,9 @@ class CropResizeParameters { /// Upscale factor (2 = 2x, 4 = 4x). final int upscaleFactor; + /// Real-CUGAN's denoise strength: none, conservative or denoise3x. + final String cuganNoise; + // --- nnedi3 controls for the edge-directed upscale paths --- // With EEDI3 selected these shape the nnedi3 sclip that guides it. @@ -245,6 +254,7 @@ class CropResizeParameters { this.useIntegerUpscale = false, this.upscaleMethod = UpscaleMethod.nnedi3Rpow2, this.upscaleFactor = 2, + this.cuganNoise = 'conservative', this.upscaleNsize, this.upscaleNeurons, this.upscaleQual, @@ -340,6 +350,7 @@ class CropResizeParameters { bool? useIntegerUpscale, UpscaleMethod? upscaleMethod, int? upscaleFactor, + String? cuganNoise, int? upscaleNsize, int? upscaleNeurons, int? upscaleQual, @@ -379,6 +390,7 @@ class CropResizeParameters { useIntegerUpscale: useIntegerUpscale ?? this.useIntegerUpscale, upscaleMethod: upscaleMethod ?? this.upscaleMethod, upscaleFactor: upscaleFactor ?? this.upscaleFactor, + cuganNoise: cuganNoise ?? this.cuganNoise, upscaleNsize: upscaleNsize ?? this.upscaleNsize, upscaleNeurons: upscaleNeurons ?? this.upscaleNeurons, upscaleQual: upscaleQual ?? this.upscaleQual, diff --git a/app/lib/models/descale_parameters.dart b/app/lib/models/descale_parameters.dart new file mode 100644 index 00000000..e521f3f7 --- /dev/null +++ b/app/lib/models/descale_parameters.dart @@ -0,0 +1,103 @@ +import 'package:json_annotation/json_annotation.dart'; + +part 'descale_parameters.g.dart'; + +/// The resampling kernel a studio upscaled with. +enum DescaleKernel { + @JsonValue('bilinear') + bilinear('Bilinear'), + @JsonValue('bicubic') + bicubic('Bicubic'), + @JsonValue('lanczos') + lanczos('Lanczos'), + @JsonValue('spline16') + spline16('Spline16'), + @JsonValue('spline36') + spline36('Spline36'), + @JsonValue('spline64') + spline64('Spline64'); + + const DescaleKernel(this.displayName); + final String displayName; +} + +/// How the studio's resampler treated pixels past the frame edge. +enum DescaleBorder { + @JsonValue('mirror') + mirror, + @JsonValue('zero') + zero, + @JsonValue('extend') + extend, +} + +/// Descale — undo a studio's upscale, then upscale again cleanly (luma only; +/// the output keeps the source's size). +@JsonSerializable() +class DescaleParameters { + final bool enabled; + + /// The height the picture was drawn at. + final int nativeHeight; + + /// The width it was drawn at; 0 derives it from the source's shape. + final int nativeWidth; + + final DescaleKernel kernel; + final double bicubicB; + final double bicubicC; + final int lanczosTaps; + final DescaleBorder border; + + /// Keep the source where re-applying the kernel does not reproduce it + /// (credits, text and overlays drawn at full resolution). + final bool protectDetail; + + /// How far (8-bit levels) the re-applied kernel may miss before a pixel + /// counts as full-resolution detail. + final double detailThreshold; + + const DescaleParameters({ + this.enabled = false, + this.nativeHeight = 720, + this.nativeWidth = 0, + this.kernel = DescaleKernel.bicubic, + this.bicubicB = 0.0, + this.bicubicC = 0.5, + this.lanczosTaps = 3, + this.border = DescaleBorder.mirror, + this.protectDetail = true, + this.detailThreshold = 4.0, + }); + + DescaleParameters copyWith({ + bool? enabled, + int? nativeHeight, + int? nativeWidth, + DescaleKernel? kernel, + double? bicubicB, + double? bicubicC, + int? lanczosTaps, + DescaleBorder? border, + bool? protectDetail, + double? detailThreshold, + }) => + DescaleParameters( + enabled: enabled ?? this.enabled, + nativeHeight: nativeHeight ?? this.nativeHeight, + nativeWidth: nativeWidth ?? this.nativeWidth, + kernel: kernel ?? this.kernel, + bicubicB: bicubicB ?? this.bicubicB, + bicubicC: bicubicC ?? this.bicubicC, + lanczosTaps: lanczosTaps ?? this.lanczosTaps, + border: border ?? this.border, + protectDetail: protectDetail ?? this.protectDetail, + detailThreshold: detailThreshold ?? this.detailThreshold, + ); + + String get summary => enabled ? '${nativeHeight}p ${kernel.displayName}' : 'Off'; + + factory DescaleParameters.fromJson(Map json) => + _$DescaleParametersFromJson(json); + Map toJson() => _$DescaleParametersToJson(this); +} diff --git a/app/lib/models/gpu_plugin_status.dart b/app/lib/models/gpu_plugin_status.dart new file mode 100644 index 00000000..0bbea1c1 --- /dev/null +++ b/app/lib/models/gpu_plugin_status.dart @@ -0,0 +1,97 @@ +/// Whether an ncnn-on-Vulkan feature runs on this computer, as reported by the +/// worker's `--probe-rife` / `--probe-upscale`. +/// +/// The worker owns the decision and the wording (`worker/src/gpu_plugin.rs`): +/// it checks the platform and the bundled files, then runs the plugin once and +/// classifies how it failed — no Vulkan device, a missing system library, an +/// older deps bundle. [GpuPluginStatus.message] is that explanation with its +/// remedy, shown to the user verbatim; the worker fails such a job with the +/// same text. +library; + +/// The two features that run a network through ncnn on the GPU. +enum GpuPlugin { + /// RIFE frame interpolation (Frame Rate pass). + rife('RIFE', '--probe-rife'), + + /// Real-ESRGAN / Real-CUGAN upscaling (Crop & Resize pass). + upscale('AI upscaling', '--probe-upscale'); + + const GpuPlugin(this.label, this.probeFlag); + + /// The name the worker's messages use. + final String label; + + /// The worker flag that probes it. + final String probeFlag; +} + +class GpuPluginStatus { + final bool available; + + /// The worker's reason id (`noVulkan`, `missingLibrary`, …), for logging. + final String? reason; + + /// What to tell the user, remedy included. Null when [available]. + final String? message; + + const GpuPluginStatus({required this.available, this.reason, this.message}); + + /// The status when the probe itself could not be run or read. + GpuPluginStatus.unknown(GpuPlugin plugin) + : available = false, + reason = 'probeFailed', + message = "VapourBox couldn't check whether ${plugin.label} runs on this " + 'computer. Restart VapourBox to try again.'; + + factory GpuPluginStatus.fromJson(GpuPlugin plugin, Map json) { + final available = json['available'] == true; + final message = json['message']; + if (!available && message is! String) return GpuPluginStatus.unknown(plugin); + return GpuPluginStatus( + available: available, + reason: json['reason'] as String?, + message: available ? null : message as String, + ); + } +} + +/// What a settings panel shows about a GPU feature's prerequisites. +enum GpuPluginNotice { + /// The feature is not selected, or it runs here. + none, + + /// The feature is selected and the probe has not answered yet. + checking, + + /// The feature is selected and cannot run here; show the status message. + unavailable, +} + +/// The one decision behind the panels' GPU notices, separate from the widget +/// so it can be tested. Shown whenever the feature is selected, enabled or +/// not: choosing it is the moment the user needs to hear it won't work. +GpuPluginNotice gpuPluginNoticeFor({ + required bool selected, + required GpuPluginStatus? status, + required bool checking, +}) { + if (!selected) return GpuPluginNotice.none; + if (checking || status == null) return GpuPluginNotice.checking; + return status.available ? GpuPluginNotice.none : GpuPluginNotice.unavailable; +} + +/// Which GPU feature, if any, a panel's current values select. +GpuPlugin? gpuPluginSelectedBy(String filterId, Map values) { + switch (filterId) { + case 'frame_rate': + return values['method'] == 'rife' ? GpuPlugin.rife : null; + case 'crop_resize': + final method = values['upscaleMethod']; + return values['useIntegerUpscale'] == true && + (method == 'realEsrgan' || method == 'realCugan') + ? GpuPlugin.upscale + : null; + } + return null; +} diff --git a/app/lib/models/native_res_report.dart b/app/lib/models/native_res_report.dart new file mode 100644 index 00000000..6357a2bc --- /dev/null +++ b/app/lib/models/native_res_report.dart @@ -0,0 +1,78 @@ +/// What `vapourbox-worker --detect-native` found: the resolution and kernel a +/// video was upscaled from, for the Descale pass (`worker/src/native_res.rs`). +class NativeResReport { + /// Whether a native resolution stands out clearly enough to use. + final bool found; + final int height; + final int width; + + /// The Descale pass's kernel id and parameters. + final String kernel; + final double bicubicB; + final double bicubicC; + final int lanczosTaps; + + /// The best height's error over its neighbours': near 0 a sharp dip. + final double dip; + + /// Every kernel tried at that height, best first. + final List<(String, double)> kernels; + + /// Why the analysis failed, when it did. + final String? error; + + const NativeResReport({ + required this.found, + required this.height, + required this.width, + required this.kernel, + this.bicubicB = 0, + this.bicubicC = 0.5, + this.lanczosTaps = 3, + this.dip = 1, + this.kernels = const [], + this.error, + }); + + const NativeResReport.failed(String this.error) + : found = false, + height = 0, + width = 0, + kernel = '', + bicubicB = 0, + bicubicC = 0.5, + lanczosTaps = 3, + dip = 1, + kernels = const []; + + factory NativeResReport.fromJson(Map json) { + if (json['error'] is String) return NativeResReport.failed(json['error'] as String); + return NativeResReport( + found: json['found'] == true, + height: (json['height'] as num?)?.toInt() ?? 0, + width: (json['width'] as num?)?.toInt() ?? 0, + kernel: json['kernel'] as String? ?? 'bicubic', + bicubicB: (json['bicubicB'] as num?)?.toDouble() ?? 0, + bicubicC: (json['bicubicC'] as num?)?.toDouble() ?? 0.5, + lanczosTaps: (json['lanczosTaps'] as num?)?.toInt() ?? 3, + dip: (json['dip'] as num?)?.toDouble() ?? 1, + kernels: [ + for (final k in (json['kernels'] as List? ?? const [])) + if (k is List && k.length == 2) (k[0].toString(), (k[1] as num).toDouble()), + ], + ); + } + + /// The best kernel's display name, e.g. "Bicubic (Catmull-Rom)". + String get kernelLabel => kernels.isEmpty ? kernel : kernels.first.$1; + + /// The values the Descale pass takes from this report. + Map get descaleValues => { + 'nativeHeight': height, + 'nativeWidth': 0, + 'kernel': kernel, + 'bicubicB': bicubicB, + 'bicubicC': bicubicC, + 'lanczosTaps': lanczosTaps, + }; +} diff --git a/app/lib/models/noise_reduction_parameters.dart b/app/lib/models/noise_reduction_parameters.dart index 015cb247..13d077ac 100644 --- a/app/lib/models/noise_reduction_parameters.dart +++ b/app/lib/models/noise_reduction_parameters.dart @@ -34,7 +34,11 @@ enum NoiseReductionMethod { @JsonValue('mClean') mClean('mClean'), @JsonValue('temporalDegrain2') - temporalDegrain2('TemporalDegrain2'); + temporalDegrain2('TemporalDegrain2'), + @JsonValue('bm3d') + bm3d('BM3D'), + @JsonValue('knlMeans') + knlMeans('KNLMeansCL'); const NoiseReductionMethod(this.displayName); final String displayName; @@ -210,8 +214,52 @@ class NoiseReductionParameters { /// Planes to filter: 0 luma only, 1 chroma only, 2 both. final int ctmfPlanes; + // --- BM3D Parameters --- + + /// Luma strength, in 8-bit units at every depth. + final double bm3dSigma; + + /// Chroma strength; 0 leaves chroma alone. + final double bm3dChromaSigma; + + /// Temporal radius (frames each side); 0 is spatial-only. + final int bm3dRadius; + + /// Run the second (Wiener) estimate: about half the speed, cleaner flats. + final bool bm3dFinal; + + /// The plugin's profile: fast, lc, np, high or vn. + final String bm3dProfile; + + // --- KNLMeansCL Parameters --- + + /// Strength. + final double knlmH; + + /// Temporal radius (frames each side); 0 is spatial-only. + final int knlmD; + + /// Search window radius. + final int knlmA; + + /// Similarity neighbourhood radius. + final int knlmS; + + /// Denoise chroma too. + final bool knlmChroma; + const NoiseReductionParameters({ this.enabled = false, + this.bm3dSigma = 3.0, + this.bm3dChromaSigma = 0.0, + this.bm3dRadius = 1, + this.bm3dFinal = true, + this.bm3dProfile = 'np', + this.knlmH = 1.2, + this.knlmD = 1, + this.knlmA = 2, + this.knlmS = 4, + this.knlmChroma = true, this.contraSharpen = false, this.mcleanStrength = 20, this.mcleanSharp = 10, @@ -359,6 +407,16 @@ class NoiseReductionParameters { double? td2PostSigma, int? td2PostMix, bool? td2ChromaMotion, + double? bm3dSigma, + double? bm3dChromaSigma, + int? bm3dRadius, + bool? bm3dFinal, + String? bm3dProfile, + double? knlmH, + int? knlmD, + int? knlmA, + int? knlmS, + bool? knlmChroma, }) { return NoiseReductionParameters( enabled: enabled ?? this.enabled, @@ -411,6 +469,16 @@ class NoiseReductionParameters { td2PostSigma: td2PostSigma ?? this.td2PostSigma, td2PostMix: td2PostMix ?? this.td2PostMix, td2ChromaMotion: td2ChromaMotion ?? this.td2ChromaMotion, + bm3dSigma: bm3dSigma ?? this.bm3dSigma, + bm3dChromaSigma: bm3dChromaSigma ?? this.bm3dChromaSigma, + bm3dRadius: bm3dRadius ?? this.bm3dRadius, + bm3dFinal: bm3dFinal ?? this.bm3dFinal, + bm3dProfile: bm3dProfile ?? this.bm3dProfile, + knlmH: knlmH ?? this.knlmH, + knlmD: knlmD ?? this.knlmD, + knlmA: knlmA ?? this.knlmA, + knlmS: knlmS ?? this.knlmS, + knlmChroma: knlmChroma ?? this.knlmChroma, ); } diff --git a/app/lib/models/parameter_converter.dart b/app/lib/models/parameter_converter.dart index 75188b4f..6d416651 100644 --- a/app/lib/models/parameter_converter.dart +++ b/app/lib/models/parameter_converter.dart @@ -2,6 +2,7 @@ import 'chroma_denoise_parameters.dart'; import 'anti_alias_parameters.dart'; import 'geometry_parameters.dart'; import 'deflicker_parameters.dart'; +import 'descale_parameters.dart'; import 'edge_repair_parameters.dart'; import 'frame_rate_parameters.dart'; import 'ghost_removal_parameters.dart'; @@ -185,6 +186,12 @@ class ParameterConverter { case NoiseReductionMethod.temporalDegrain2: method = 'temporal_degrain2'; break; + case NoiseReductionMethod.bm3d: + method = 'bm3d'; + break; + case NoiseReductionMethod.knlMeans: + method = 'knlmeans'; + break; } return DynamicParameters( @@ -238,6 +245,16 @@ class ParameterConverter { 'stpressoTthr': params.stpressoTthr, 'ctmfRadius': params.ctmfRadius, 'ctmfPlanes': params.ctmfPlanes, + 'bm3dSigma': params.bm3dSigma, + 'bm3dChromaSigma': params.bm3dChromaSigma, + 'bm3dRadius': params.bm3dRadius, + 'bm3dFinal': params.bm3dFinal, + 'bm3dProfile': params.bm3dProfile, + 'knlmH': params.knlmH, + 'knlmD': params.knlmD, + 'knlmA': params.knlmA, + 'knlmS': params.knlmS, + 'knlmChroma': params.knlmChroma, }, ); } @@ -437,6 +454,44 @@ class ParameterConverter { /// Convert grain parameters to dynamic format. /// Convert frame rate parameters to dynamic format. + /// Convert descale parameters to dynamic format. + static DynamicParameters fromDescale(DescaleParameters params) { + return DynamicParameters( + filterId: 'descale', + enabled: params.enabled, + values: { + 'nativeHeight': params.nativeHeight, + 'nativeWidth': params.nativeWidth, + 'kernel': params.kernel.name, + 'bicubicB': params.bicubicB, + 'bicubicC': params.bicubicC, + 'lanczosTaps': params.lanczosTaps, + 'border': params.border.name, + 'protectDetail': params.protectDetail, + 'detailThreshold': params.detailThreshold, + }, + ); + } + + /// Convert dynamic parameters to descale parameters. + static DescaleParameters toDescale(DynamicParameters params) { + final v = params.values; + return DescaleParameters( + enabled: params.enabled, + nativeHeight: _asInt(v['nativeHeight']) ?? 720, + nativeWidth: _asInt(v['nativeWidth']) ?? 0, + kernel: DescaleKernel.values.firstWhere((k) => k.name == v['kernel'], + orElse: () => DescaleKernel.bicubic), + bicubicB: (v['bicubicB'] as num?)?.toDouble() ?? 0.0, + bicubicC: (v['bicubicC'] as num?)?.toDouble() ?? 0.5, + lanczosTaps: _asInt(v['lanczosTaps']) ?? 3, + border: DescaleBorder.values.firstWhere((b) => b.name == v['border'], + orElse: () => DescaleBorder.mirror), + protectDetail: v['protectDetail'] as bool? ?? true, + detailThreshold: (v['detailThreshold'] as num?)?.toDouble() ?? 4.0, + ); + } + /// Convert deflicker parameters to dynamic format. static DynamicParameters fromDeflicker(DeflickerParameters params) { return DynamicParameters( @@ -860,6 +915,7 @@ class ParameterConverter { 'useIntegerUpscale': params.useIntegerUpscale, 'upscaleMethod': params.upscaleMethod.name, 'upscaleFactor': params.upscaleFactor, + 'cuganNoise': params.cuganNoise, }; // Kernel tuning and the nnedi3/EEDI3 controls start unticked, so the // plugin's own default applies until the user opts in. Fallbacks are those @@ -926,6 +982,7 @@ class ParameterConverter { 'grain': fromGrain(pipeline.grain), 'frame_rate': fromFrameRate(pipeline.frameRate), 'deflicker': fromDeflicker(pipeline.deflicker), + 'descale': fromDescale(pipeline.descale), 'edge_repair': fromEdgeRepair(pipeline.edgeRepair), 'ghost_removal': fromGhostRemoval(pipeline.ghostRemoval), 'chroma_denoise': fromChromaDenoise(pipeline.chromaDenoise), @@ -1105,6 +1162,12 @@ class ParameterConverter { case 'temporal_degrain2': method = NoiseReductionMethod.temporalDegrain2; break; + case 'bm3d': + method = NoiseReductionMethod.bm3d; + break; + case 'knlmeans': + method = NoiseReductionMethod.knlMeans; + break; default: method = NoiseReductionMethod.smDegrain; } @@ -1159,6 +1222,16 @@ class ParameterConverter { stpressoTthr: _asInt(v['stpressoTthr']) ?? 12, ctmfRadius: _asInt(v['ctmfRadius']) ?? 2, ctmfPlanes: _asInt(v['ctmfPlanes']) ?? 2, + bm3dSigma: (v['bm3dSigma'] as num?)?.toDouble() ?? 3.0, + bm3dChromaSigma: (v['bm3dChromaSigma'] as num?)?.toDouble() ?? 0.0, + bm3dRadius: _asInt(v['bm3dRadius']) ?? 1, + bm3dFinal: v['bm3dFinal'] as bool? ?? true, + bm3dProfile: v['bm3dProfile'] as String? ?? 'np', + knlmH: (v['knlmH'] as num?)?.toDouble() ?? 1.2, + knlmD: _asInt(v['knlmD']) ?? 1, + knlmA: _asInt(v['knlmA']) ?? 2, + knlmS: _asInt(v['knlmS']) ?? 4, + knlmChroma: v['knlmChroma'] as bool? ?? true, ); } @@ -1470,7 +1543,8 @@ class ParameterConverter { (m) => m.name.toLowerCase() == (v['upscaleMethod'] as String? ?? 'nnedi3Rpow2').toLowerCase(), orElse: () => UpscaleMethod.nnedi3Rpow2, ), - upscaleFactor: v['upscaleFactor'] as int? ?? 2, + upscaleFactor: _asInt(v['upscaleFactor']) ?? 2, + cuganNoise: v['cuganNoise'] as String? ?? 'conservative', // Absent (checkbox off) stays null so the script omits it entirely. bicubicB: (v['bicubicB'] as num?)?.toDouble(), bicubicC: (v['bicubicC'] as num?)?.toDouble(), @@ -1531,6 +1605,9 @@ class ParameterConverter { deflicker: dynamic.get('deflicker') != null ? toDeflicker(dynamic.get('deflicker')!) : const DeflickerParameters(), + descale: dynamic.get('descale') != null + ? toDescale(dynamic.get('descale')!) + : const DescaleParameters(), edgeRepair: dynamic.get('edge_repair') != null ? toEdgeRepair(dynamic.get('edge_repair')!) : const EdgeRepairParameters(), diff --git a/app/lib/models/pass_relevance.dart b/app/lib/models/pass_relevance.dart index 7baeaf26..33e4b380 100644 --- a/app/lib/models/pass_relevance.dart +++ b/app/lib/models/pass_relevance.dart @@ -152,6 +152,9 @@ PassRelevanceResult relevanceFor(PassType pass, VideoInfo? info) { case PassType.deflicker: case PassType.edgeRepair: case PassType.ghostRemoval: + // Whether a picture was upscaled from a lower resolution is not in the + // metadata at all; only the pass's own detection can tell. + case PassType.descale: case PassType.frameRate: case PassType.grain: return PassRelevanceResult.neutral; diff --git a/app/lib/models/processing_pipeline.dart b/app/lib/models/processing_pipeline.dart index 66f3bcd0..8882454f 100644 --- a/app/lib/models/processing_pipeline.dart +++ b/app/lib/models/processing_pipeline.dart @@ -3,6 +3,7 @@ import 'package:json_annotation/json_annotation.dart'; import 'anti_alias_parameters.dart'; import 'geometry_parameters.dart'; import 'deflicker_parameters.dart'; +import 'descale_parameters.dart'; import 'edge_repair_parameters.dart'; import 'frame_rate_parameters.dart'; import 'ghost_removal_parameters.dart'; @@ -45,6 +46,7 @@ enum PassType { deflicker, edgeRepair, ghostRemoval, + descale, colorCorrection, chromaFixes, cropResize, @@ -89,6 +91,8 @@ extension PassTypeExtension on PassType { return 'Edge Repair'; case PassType.ghostRemoval: return 'Ghost Removal'; + case PassType.descale: + return 'Descale'; case PassType.colorCorrection: return 'Color Correction'; case PassType.chromaFixes: @@ -136,6 +140,8 @@ extension PassTypeExtension on PassType { return 'Rebuild the dirty rows and columns at the frame border'; case PassType.ghostRemoval: return 'Remove the displaced echo RF and cable distribution leave behind'; + case PassType.descale: + return "Undo a studio's upscale, then upscale again cleanly"; case PassType.colorCorrection: return 'Adjust brightness, contrast, and colors'; case PassType.chromaFixes: @@ -184,6 +190,7 @@ class ProcessingPipeline { final GrainParameters grain; final FrameRateParameters frameRate; final DeflickerParameters deflicker; + final DescaleParameters descale; final EdgeRepairParameters edgeRepair; final GhostRemovalParameters ghostRemoval; @@ -215,6 +222,7 @@ class ProcessingPipeline { this.grain = const GrainParameters(), this.frameRate = const FrameRateParameters(), this.deflicker = const DeflickerParameters(), + this.descale = const DescaleParameters(), this.edgeRepair = const EdgeRepairParameters(), this.ghostRemoval = const GhostRemovalParameters(), this.colorCorrection = const ColorCorrectionParameters(), @@ -242,6 +250,7 @@ class ProcessingPipeline { grain: const GrainParameters(enabled: false), frameRate: const FrameRateParameters(enabled: false), deflicker: const DeflickerParameters(enabled: false), + descale: const DescaleParameters(enabled: false), edgeRepair: const EdgeRepairParameters(enabled: false), ghostRemoval: const GhostRemovalParameters(enabled: false), colorCorrection: const ColorCorrectionParameters(enabled: false), @@ -281,6 +290,11 @@ class ProcessingPipeline { if (spotless.enabled) { passes.add(PassType.spotless); } + // Descale needs the studio's upscale exactly as it was: before any denoise, + // deblock or sharpen changes the picture it inverts. + if (descale.enabled) { + passes.add(PassType.descale); + } if (noiseReduction.enabled) { passes.add(PassType.noiseReduction); } @@ -366,6 +380,7 @@ class ProcessingPipeline { if (grain.hasEffect) count++; if (frameRate.enabled) count++; if (deflicker.enabled) count++; + if (descale.enabled) count++; if (edgeRepair.hasEffect) count++; if (ghostRemoval.hasEffect) count++; if (colorCorrection.enabled) count++; @@ -393,6 +408,7 @@ class ProcessingPipeline { if (grain.hasEffect) count++; if (frameRate.enabled) count++; if (deflicker.enabled) count++; + if (descale.enabled) count++; if (edgeRepair.hasEffect) count++; if (ghostRemoval.hasEffect) count++; if (colorCorrection.enabled) count++; @@ -434,6 +450,8 @@ class ProcessingPipeline { return frameRate.enabled; case PassType.deflicker: return deflicker.enabled; + case PassType.descale: + return descale.enabled; case PassType.edgeRepair: return edgeRepair.hasEffect; case PassType.ghostRemoval: @@ -484,6 +502,8 @@ class ProcessingPipeline { return frameRate.summary; case PassType.deflicker: return deflicker.summary; + case PassType.descale: + return descale.summary; case PassType.edgeRepair: return edgeRepair.summary; case PassType.ghostRemoval: @@ -515,6 +535,7 @@ class ProcessingPipeline { GrainParameters? grain, FrameRateParameters? frameRate, DeflickerParameters? deflicker, + DescaleParameters? descale, EdgeRepairParameters? edgeRepair, GhostRemovalParameters? ghostRemoval, ColorCorrectionParameters? colorCorrection, @@ -538,6 +559,7 @@ class ProcessingPipeline { grain: grain ?? this.grain, frameRate: frameRate ?? this.frameRate, deflicker: deflicker ?? this.deflicker, + descale: descale ?? this.descale, edgeRepair: edgeRepair ?? this.edgeRepair, ghostRemoval: ghostRemoval ?? this.ghostRemoval, colorCorrection: colorCorrection ?? this.colorCorrection, @@ -608,6 +630,8 @@ class ProcessingPipeline { ); case PassType.deflicker: return copyWith(deflicker: deflicker.copyWith(enabled: enabled)); + case PassType.descale: + return copyWith(descale: descale.copyWith(enabled: enabled)); case PassType.edgeRepair: return copyWith(edgeRepair: edgeRepair.copyWith(enabled: enabled)); case PassType.ghostRemoval: diff --git a/app/lib/models/rife_status.dart b/app/lib/models/rife_status.dart deleted file mode 100644 index 0fccc3e6..00000000 --- a/app/lib/models/rife_status.dart +++ /dev/null @@ -1,62 +0,0 @@ -/// Whether RIFE frame interpolation runs on this computer, as reported by -/// `vapourbox-worker --probe-rife`. -/// -/// The worker owns the decision and the wording (`worker/src/rife.rs`): it -/// checks the platform and the bundled files, then runs the plugin once and -/// classifies how it failed — no Vulkan device, a missing system library, an -/// older deps bundle. [message] is that explanation with its remedy, shown to -/// the user verbatim; the worker fails a RIFE job with the same text. -class RifeStatus { - final bool available; - - /// The worker's reason id (`noVulkan`, `missingLibrary`, …), for logging. - final String? reason; - - /// What to tell the user, remedy included. Null when [available]. - final String? message; - - const RifeStatus({required this.available, this.reason, this.message}); - - /// The status when the probe itself could not be run or read. - const RifeStatus.unknown() - : available = false, - reason = 'probeFailed', - message = "VapourBox couldn't check whether RIFE runs on this " - 'computer. Restart VapourBox to try again.'; - - factory RifeStatus.fromJson(Map json) { - final available = json['available'] == true; - final message = json['message']; - if (!available && message is! String) return const RifeStatus.unknown(); - return RifeStatus( - available: available, - reason: json['reason'] as String?, - message: available ? null : message as String, - ); - } -} - -/// What the Frame Rate panel shows about RIFE's prerequisites. -enum RifeNotice { - /// RIFE is not selected, or it runs here. - none, - - /// RIFE is selected and the probe has not answered yet. - checking, - - /// RIFE is selected and cannot run here; show [RifeStatus.message]. - unavailable, -} - -/// The one decision behind the panel's RIFE notice, separate from the widget -/// so it can be tested. Shown whenever RIFE is the selected method, enabled or -/// not: choosing it is the moment the user needs to hear it won't work. -RifeNotice rifeNoticeFor({ - required Object? method, - required RifeStatus? status, - required bool checking, -}) { - if (method != 'rife') return RifeNotice.none; - if (checking || status == null) return RifeNotice.checking; - return status.available ? RifeNotice.none : RifeNotice.unavailable; -} diff --git a/app/lib/services/worker_manager.dart b/app/lib/services/worker_manager.dart index 5cd56ca3..ecde1c15 100644 --- a/app/lib/services/worker_manager.dart +++ b/app/lib/services/worker_manager.dart @@ -2,10 +2,10 @@ import 'dart:async'; import 'dart:convert'; import 'dart:io'; -import 'package:flutter/foundation.dart'; import '../models/progress_info.dart'; -import '../models/rife_status.dart'; +import '../models/gpu_plugin_status.dart'; +import '../models/native_res_report.dart'; import '../models/video_job.dart'; import 'job_runner.dart'; import 'process_tree.dart'; @@ -177,13 +177,23 @@ class WorkerManager implements JobRunner { /// - `opencl`: a usable OpenCL device for the QTGMC NNEDI3CL path. /// - `knlm`: the knlmeanscl denoiser specifically (a DISTINCT probe — /// KNLMeansCL can fail where NNEDI3CL succeeds). + /// - `knlmMessage`: why the KNLMeans noise-reduction method cannot run, or + /// null when it can (it runs on whichever device the worker found, which is + /// not the question `knlm` answers). /// /// Both default to `false` if the worker can't be found, fails, or doesn't /// report the field — callers treat that as "warn the user", the safe default. - static Future<({bool opencl, bool knlm})> probeGpuCapabilities() async { + static Future<({bool opencl, bool knlm, String? knlmMessage})> + probeGpuCapabilities() async { + const none = ( + opencl: false, + knlm: false, + knlmMessage: 'VapourBox could not check for an OpenCL device, which ' + 'KNLMeans needs. BM3D does the same job on the CPU.', + ); final toolLocator = ToolLocator.instance; final workerPath = toolLocator.workerPath; - if (workerPath == null) return (opencl: false, knlm: false); + if (workerPath == null) return none; try { final result = await Process.run( @@ -192,7 +202,7 @@ class WorkerManager implements JobRunner { environment: toolLocator.workerEnvironment, workingDirectory: File(workerPath).parent.path, ); - if (result.exitCode != 0) return (opencl: false, knlm: false); + if (result.exitCode != 0) return none; // The worker prints a single JSON line; tolerate extra log lines. for (final line in const LineSplitter().convert(result.stdout.toString())) { final trimmed = line.trim(); @@ -203,53 +213,94 @@ class WorkerManager implements JobRunner { return ( opencl: decoded['opencl'] as bool, knlm: decoded['knlm'] == true, + knlmMessage: decoded['knlmDevice'] is String + ? null + : (decoded['knlmMessage'] as String? ?? none.knlmMessage), ); } } catch (_) { // Not the JSON line; keep scanning. } } - return (opencl: false, knlm: false); + return none; } catch (_) { - return (opencl: false, knlm: false); + return none; } } - /// Ask the worker whether RIFE frame interpolation runs here - /// (`--probe-rife`). The first call after a boot initialises the GPU, so it - /// takes a second or two; a positive answer is cached by the worker. + /// Ask the worker whether a GPU feature (RIFE, AI upscaling) runs here + /// (`--probe-rife` / `--probe-upscale`). The first call after a boot + /// initialises the GPU, so it takes a second or two; a positive answer is + /// cached by the worker. /// /// Never throws: a worker that can't be run or answers with something - /// unreadable is reported as [RifeStatus.unknown], which tells the user to - /// restart rather than claiming RIFE works. - static Future probeRife() async { + /// unreadable is reported as [GpuPluginStatus.unknown], which tells the user + /// to restart rather than claiming the feature works. + static Future probeGpuPlugin(GpuPlugin plugin) async { final toolLocator = ToolLocator.instance; final workerPath = toolLocator.workerPath; - if (workerPath == null) return const RifeStatus.unknown(); + if (workerPath == null) return GpuPluginStatus.unknown(plugin); try { final result = await Process.run( workerPath, - ['--probe-rife'], + [plugin.probeFlag], environment: toolLocator.workerEnvironment, workingDirectory: File(workerPath).parent.path, ); - if (result.exitCode != 0) return const RifeStatus.unknown(); + if (result.exitCode != 0) return GpuPluginStatus.unknown(plugin); for (final line in const LineSplitter().convert(result.stdout.toString())) { final trimmed = line.trim(); if (trimmed.isEmpty) continue; try { final decoded = jsonDecode(trimmed); if (decoded is Map && decoded['available'] is bool) { - return RifeStatus.fromJson(decoded); + return GpuPluginStatus.fromJson(plugin, decoded); } } catch (_) { // Not the JSON line; keep scanning. } } - return const RifeStatus.unknown(); + return GpuPluginStatus.unknown(plugin); } catch (_) { - return const RifeStatus.unknown(); + return GpuPluginStatus.unknown(plugin); + } + } + + /// Find the resolution and kernel [job]'s video was upscaled from + /// (`--detect-native`). Decodes a few frames and tries every height and the + /// common kernels, so it takes tens of seconds on HD. Never throws: a failure + /// comes back as [NativeResReport.failed]. + static Future detectNativeResolution(VideoJob job) async { + final toolLocator = ToolLocator.instance; + final workerPath = toolLocator.workerPath; + if (workerPath == null) { + return const NativeResReport.failed('The VapourBox worker could not be found.'); + } + final config = File(await TempDirectoryService.instance + .filePath('detect_native_${job.id}.json')); + try { + await config.writeAsString(jsonEncode(job.toJson())); + final result = await Process.run( + workerPath, + ['--detect-native', '--config', config.path], + environment: toolLocator.workerEnvironment, + workingDirectory: File(workerPath).parent.path, + ); + for (final line in const LineSplitter().convert(result.stdout.toString()).reversed) { + final trimmed = line.trim(); + if (!trimmed.startsWith('{')) continue; + try { + return NativeResReport.fromJson(jsonDecode(trimmed) as Map); + } catch (_) { + // Not the JSON line; keep scanning. + } + } + return NativeResReport.failed('The analysis did not report a result: ${result.stderr}'); + } catch (e) { + return NativeResReport.failed('$e'); + } finally { + if (await config.exists()) await config.delete(); } } diff --git a/app/lib/viewmodels/main_viewmodel.dart b/app/lib/viewmodels/main_viewmodel.dart index aa7fcf74..ac19c17b 100644 --- a/app/lib/viewmodels/main_viewmodel.dart +++ b/app/lib/viewmodels/main_viewmodel.dart @@ -5,7 +5,8 @@ import 'package:path/path.dart' as path; import '../models/dvd_info.dart'; import '../models/dynamic_parameters.dart'; -import '../models/rife_status.dart'; +import '../models/gpu_plugin_status.dart'; +import '../models/native_res_report.dart'; import '../models/encoding_settings.dart'; import '../models/parameter_converter.dart'; import '../models/progress_info.dart'; @@ -84,11 +85,20 @@ class MainViewModel extends ChangeNotifier { // (a distinct capability — KNLMeansCL can fail where NNEDI3CL succeeds). bool? _openclAvailable; bool? _knlmAvailable; + // Why the KNLMeans noise-reduction method cannot run here (null when it can, + // or before the probe has answered). + String? _knlmMethodMessage; - // Whether RIFE frame interpolation runs here (`--probe-rife`), with the - // worker's explanation and remedy when it doesn't. null = not yet probed. - RifeStatus? _rifeStatus; - bool _rifeChecking = false; + // Whether each GPU feature (RIFE, AI upscaling) runs here, with the worker's + // explanation and remedy when it doesn't. Absent = not yet probed. + final Map _gpuStatus = {}; + final Set _gpuChecking = {}; + + // Descale's Detect: the native resolution found for a video, keyed by the + // queue item it was measured on, so it never applies to a different one. + final Future Function(VideoJob job) _detectNative; + ({String itemId, NativeResReport report})? _nativeRes; + String? _detectingNativeResFor; // Set in dispose() so the async OpenCL probe doesn't notifyListeners() after // teardown (the top-level view model normally lives for the app's lifetime). @@ -172,35 +182,121 @@ class MainViewModel extends ChangeNotifier { /// while the probe is still running. bool? get knlmAvailable => _knlmAvailable; - /// Whether RIFE runs here, and if not why (gates the Frame Rate panel's - /// RIFE notice). null while the first probe is still running. - RifeStatus? get rifeStatus => _rifeStatus; + /// Why the KNLMeans noise-reduction method cannot run here — the worker's + /// wording, the same text it refuses such a job with — or null. + String? get knlmMethodMessage => _knlmMethodMessage; + + @visibleForTesting + void debugSetKnlmMethodMessage(String? message) { + _knlmMethodMessage = message; + notifyListeners(); + } - /// Whether a RIFE probe is in flight. - bool get rifeChecking => _rifeChecking; + /// Whether a GPU feature runs here, and if not why (gates the panels' + /// notices). null while the first probe is still running. + GpuPluginStatus? gpuPluginStatus(GpuPlugin plugin) => _gpuStatus[plugin]; - /// Set the RIFE probe result directly, for widget tests of the notice. + /// Whether a probe for [plugin] is in flight. + bool gpuPluginChecking(GpuPlugin plugin) => _gpuChecking.contains(plugin); + + /// Set a probe result directly, for widget tests of the notices. @visibleForTesting - void debugSetRifeStatus(RifeStatus? status, {bool checking = false}) { - _rifeStatus = status; - _rifeChecking = checking; + void debugSetGpuPluginStatus(GpuPlugin plugin, GpuPluginStatus? status, + {bool checking = false}) { + status == null ? _gpuStatus.remove(plugin) : _gpuStatus[plugin] = status; + checking ? _gpuChecking.add(plugin) : _gpuChecking.remove(plugin); notifyListeners(); } - /// Re-run the RIFE probe — after the user has installed a driver or the - /// VC++ runtime the notice told them to. The worker only caches a positive - /// answer, so this sees the fix without a reboot. - Future recheckRife() async { - if (_rifeChecking) return; - _rifeChecking = true; + /// Re-run a GPU feature's probe — after the user has installed a driver or + /// the VC++ runtime the notice told them to. The worker only caches a + /// positive answer, so this sees the fix without a reboot. + Future recheckGpuPlugin(GpuPlugin plugin) async { + if (_gpuChecking.contains(plugin)) return; + _gpuChecking.add(plugin); notifyListeners(); - final status = await WorkerManager.probeRife(); + final status = await WorkerManager.probeGpuPlugin(plugin); if (_disposed) return; - _rifeStatus = status; - _rifeChecking = false; + _gpuStatus[plugin] = status; + _gpuChecking.remove(plugin); + notifyListeners(); + } + + /// Put an analysed video in the queue and select it, for widget tests. + @visibleForTesting + void debugAddQueueItemForTest(String path, {required int width, required int height}) { + final item = QueueItem( + inputPath: path, + outputPath: '$path.out.mkv', + videoInfo: VideoInfo( + width: width, + height: height, + frameRate: 23.976, + duration: 600, + frameCount: 14386, + codec: 'h264', + pixelFormat: 'yuv420p', + hasAudio: false, + ), + ); + _queue.add(item); + _selectedItemId = item.id; notifyListeners(); } + /// What Detect found for the selected video, or null (not run for it yet). + NativeResReport? get nativeResReport => + _nativeRes != null && _nativeRes!.itemId == _selectedItemId ? _nativeRes!.report : null; + + /// Whether Detect is running for the selected video. + bool get detectingNativeRes => + _detectingNativeResFor != null && _detectingNativeResFor == _selectedItemId; + + /// Find the resolution and kernel the selected video was upscaled from, for + /// the Descale pass. Analyses the trimmed range if there is one. + Future detectNativeResolution() async { + final item = selectedItem; + if (item == null || _detectingNativeResFor != null) return; + _detectingNativeResFor = item.id; + notifyListeners(); + final info = item.videoInfo; + final frameCount = info?.frameCount ?? 0; + final job = VideoJob( + inputPath: item.inputPath, + outputPath: item.outputPath, + encodingSettings: _encodingSettings, + totalFrames: info?.frameCount, + inputFrameRate: info?.frameRate, + inputWidth: info?.width, + inputHeight: info?.height, + inputPixelFormat: info?.pixelFormat, + startFrame: item.inPoint == null ? null : (item.inPoint! * frameCount).round(), + endFrame: item.outPoint == null ? null : (item.outPoint! * frameCount).round(), + ); + final report = await _detectNative(job); + if (_disposed) return; + _nativeRes = (itemId: item.id, report: report); + _detectingNativeResFor = null; + notifyListeners(); + } + + /// Put what Detect found into the Descale pass, switching it on. + void applyNativeResolution() { + final report = nativeResReport; + if (report == null || report.error != null || report.height <= 0) return; + final current = getDynamicParams('descale'); + updateDynamicParams( + 'descale', + DynamicParameters( + filterId: 'descale', + enabled: true, + values: {...current.values, ...report.descaleValues}, + enabledOptionalParams: current.enabledOptionalParams, + lastOptionalValues: current.lastOptionalValues, + ), + ); + } + bool get canShutDown => _systemShutdown.isSupported; bool get shutdownWhenDone => _shutdownWhenDone; @@ -280,6 +376,8 @@ class MainViewModel extends ChangeNotifier { return ParameterConverter.fromEdgeRepair(_processingPipeline.edgeRepair); case 'ghost_removal': return ParameterConverter.fromGhostRemoval(_processingPipeline.ghostRemoval); + case 'descale': + return ParameterConverter.fromDescale(_processingPipeline.descale); case 'color_correction': return ParameterConverter.fromColorCorrection(_processingPipeline.colorCorrection); case 'chroma_fixes': @@ -366,6 +464,11 @@ class MainViewModel extends ChangeNotifier { deflicker: ParameterConverter.toDeflicker(params), ); break; + case 'descale': + _processingPipeline = _processingPipeline.copyWith( + descale: ParameterConverter.toDescale(params), + ); + break; case 'edge_repair': _processingPipeline = _processingPipeline.copyWith( edgeRepair: ParameterConverter.toEdgeRepair(params), @@ -503,13 +606,19 @@ class MainViewModel extends ChangeNotifier { return _manualFieldOrder; } - MainViewModel({JobRunner? runner, SystemShutdown? systemShutdown}) - : _workerManager = runner ?? WorkerManager(), - _systemShutdown = systemShutdown ?? PlatformSystemShutdown() { + MainViewModel({ + JobRunner? runner, + SystemShutdown? systemShutdown, + Future Function(VideoJob job)? nativeResDetector, + }) : _workerManager = runner ?? WorkerManager(), + _systemShutdown = systemShutdown ?? PlatformSystemShutdown(), + _detectNative = nativeResDetector ?? WorkerManager.detectNativeResolution { _setupSubscriptions(); _initializePreviewGenerator(); _probeGpuCapabilities(); - recheckRife(); + for (final plugin in GpuPlugin.values) { + recheckGpuPlugin(plugin); + } } /// Probe GPU capabilities once at startup so the UI can warn when an @@ -519,6 +628,7 @@ class MainViewModel extends ChangeNotifier { if (_disposed) return; _openclAvailable = caps.opencl; _knlmAvailable = caps.knlm; + _knlmMethodMessage = caps.knlmMessage; notifyListeners(); } @@ -1225,6 +1335,8 @@ class MainViewModel extends ChangeNotifier { return 'edge_repair'; case PassType.ghostRemoval: return 'ghost_removal'; + case PassType.descale: + return 'descale'; case PassType.colorCorrection: return 'color_correction'; case PassType.chromaFixes: diff --git a/app/lib/views/about_dialog.dart b/app/lib/views/about_dialog.dart index 441db5a6..593cc904 100644 --- a/app/lib/views/about_dialog.dart +++ b/app/lib/views/about_dialog.dart @@ -286,6 +286,26 @@ class _AboutDialogState extends State { copyright: 'The Brenwill Workshop Ltd. (macOS only)', url: 'https://github.com/KhronosGroup/MoltenVK', ), + _ComponentTile( + name: 'VSNCNN (vs-mlrt)', + license: 'GPL-3.0', + copyright: 'AmusementClub (no copyright notice upstream); ' + 'includes ncnn (Tencent), Protocol Buffers (Google, ' + 'BSD 3-Clause) and ONNX (Apache 2.0)', + url: 'https://github.com/AmusementClub/vs-mlrt', + ), + _ComponentTile( + name: 'Real-ESRGAN anime video model', + license: 'BSD 3-Clause', + copyright: 'Xintao Wang', + url: 'https://github.com/xinntao/Real-ESRGAN', + ), + _ComponentTile( + name: 'Real-CUGAN models', + license: 'MIT', + copyright: 'bilibili', + url: 'https://github.com/bilibili/ailab/tree/main/Real-CUGAN', + ), _ComponentTile( name: 'DFTTest', license: 'GPL 3.0', @@ -360,6 +380,12 @@ class _AboutDialogState extends State { copyright: 'mawen1250', url: 'https://github.com/HomeOfVapourSynthEvolution/VapourSynth-BM3D', ), + _ComponentTile( + name: 'descale', + license: 'MIT', + copyright: 'Frechdachs; maintained by Jaded Encoding Thaumaturgy', + url: 'https://github.com/Jaded-Encoding-Thaumaturgy/vapoursynth-descale', + ), _ComponentTile( name: 'KNLMeansCL', license: 'GPL 3.0', diff --git a/app/lib/views/pass_list/pass_list_item.dart b/app/lib/views/pass_list/pass_list_item.dart index 1c039eaa..7c1f4412 100644 --- a/app/lib/views/pass_list/pass_list_item.dart +++ b/app/lib/views/pass_list/pass_list_item.dart @@ -255,6 +255,8 @@ class PassListItem extends StatelessWidget { return Icons.border_outer; case PassType.ghostRemoval: return Icons.blur_linear; + case PassType.descale: + return Icons.photo_size_select_large; case PassType.colorCorrection: return Icons.palette; case PassType.chromaFixes: diff --git a/app/lib/views/pass_list/pass_list_panel.dart b/app/lib/views/pass_list/pass_list_panel.dart index 73caa106..f67e61c7 100644 --- a/app/lib/views/pass_list/pass_list_panel.dart +++ b/app/lib/views/pass_list/pass_list_panel.dart @@ -47,6 +47,7 @@ class PassListPanel extends StatelessWidget { ( title: 'Noise & Artifacts', passes: [ + PassType.descale, PassType.noiseReduction, PassType.chromaDenoise, PassType.dehalo, @@ -156,6 +157,9 @@ class PassListPanel extends StatelessWidget { case PassType.deflicker: return item(passType, 'Deflicker', pipeline.deflicker.summary, pipeline.deflicker.enabled); + case PassType.descale: + return item(passType, 'Descale', pipeline.descale.summary, + pipeline.descale.enabled); case PassType.edgeRepair: return item(passType, 'Edge Repair', pipeline.edgeRepair.summary, pipeline.edgeRepair.enabled); diff --git a/app/lib/views/pass_settings/pass_settings_inline.dart b/app/lib/views/pass_settings/pass_settings_inline.dart index 52b75042..ff101612 100644 --- a/app/lib/views/pass_settings/pass_settings_inline.dart +++ b/app/lib/views/pass_settings/pass_settings_inline.dart @@ -6,7 +6,7 @@ import '../../models/filter_registry.dart'; import '../../models/filter_schema.dart'; import '../../models/pass_advice.dart'; import '../../models/processing_pipeline.dart'; -import '../../models/rife_status.dart'; +import '../../models/gpu_plugin_status.dart'; import '../../services/whisper_addon_manager.dart'; import '../../utils/pixel_format.dart'; import '../../viewmodels/main_viewmodel.dart'; @@ -65,6 +65,8 @@ class PassSettingsInline extends StatelessWidget { return 'edge_repair'; case PassType.ghostRemoval: return 'ghost_removal'; + case PassType.descale: + return 'descale'; case PassType.colorCorrection: return 'color_correction'; case PassType.chromaFixes: @@ -97,7 +99,9 @@ class PassSettingsInline extends StatelessWidget { _buildDescription(context, schema), _buildInteractionAdvice(context, viewModel), _buildOpenCLWarning(context, viewModel, filterId, params), - _buildRifeNotice(context, viewModel, filterId, params), + _buildGpuPluginNotice(context, viewModel, filterId, params), + _buildKnlmNotice(viewModel, filterId, params), + if (filterId == 'descale') _buildDescaleDetect(context, viewModel), _buildBitDepthWarning(context, viewModel, schema, params), DynamicFilterPanelCompact( schema: schema, @@ -161,27 +165,118 @@ class PassSettingsInline extends StatelessWidget { return WarningBanner(message: message); } - /// RIFE's prerequisites, the moment it is chosen. Unlike the OpenCL options - /// there is no fallback — a RIFE job on a machine without a Vulkan device - /// is refused — so this says exactly what is missing and how to fix it (the - /// worker's wording, see `rife.rs`), with a button to check again once the - /// user has. - Widget _buildRifeNotice( + /// Descale's Detect: run the analysis on the selected video and offer what + /// it found. A wrong native resolution damages the picture, so a source with + /// no clear one is told so plainly rather than handed a guess to apply. + Widget _buildDescaleDetect(BuildContext context, MainViewModel viewModel) { + final theme = Theme.of(context); + final report = viewModel.nativeResReport; + final running = viewModel.detectingNativeRes; + final canRun = viewModel.selectedItem != null && !running; + final children = [ + Row( + children: [ + OutlinedButton.icon( + onPressed: canRun ? viewModel.detectNativeResolution : null, + icon: running + ? const SizedBox.square( + dimension: 14, child: CircularProgressIndicator(strokeWidth: 2)) + : const Icon(Icons.search, size: 16), + label: Text(running ? 'Detecting…' : 'Detect native resolution'), + ), + const SizedBox(width: 12), + Expanded( + child: Text( + running + ? 'Trying every height and the common kernels on frames from ' + 'this video. About half a minute for HD.' + : viewModel.selectedItem == null + ? 'Select a video to analyse.' + : 'Finds the resolution and kernel this video was upscaled from.', + style: theme.textTheme.bodySmall, + ), + ), + ], + ), + ]; + if (!running && report != null) { + if (report.error != null) { + children.add(WarningBanner(message: 'Detection failed: ${report.error}')); + } else if (report.found) { + children.add(Padding( + padding: const EdgeInsets.only(top: 8), + child: Row( + children: [ + Icon(Icons.check_circle, size: 18, color: theme.colorScheme.primary), + const SizedBox(width: 8), + Expanded( + child: SelectableText( + 'Drawn at ${report.width}×${report.height}, upscaled with ' + '${report.kernelLabel}.', + ), + ), + FilledButton( + onPressed: viewModel.applyNativeResolution, + child: const Text('Use these settings'), + ), + ], + ), + )); + } else { + children.add(WarningBanner( + message: 'No clear native resolution. This video does not look like ' + 'it was upscaled from a lower one, and descaling it would soften ' + 'it. (Closest candidate: ${report.height}p, ' + '${report.kernelLabel}.)', + )); + } + } + return Padding( + padding: const EdgeInsets.only(bottom: 12), + child: Column(crossAxisAlignment: CrossAxisAlignment.stretch, children: children), + ); + } + + /// Why the KNLMeans noise-reduction method can't run here, the moment it is + /// chosen. Like RIFE it has no fallback (the worker refuses the job), so the + /// worker's own reason is shown verbatim. + Widget _buildKnlmNotice( + MainViewModel viewModel, + String filterId, + DynamicParameters params, + ) { + final message = viewModel.knlmMethodMessage; + if (filterId != 'noise_reduction' || + params.values['method'] != 'knlmeans' || + message == null) { + return const SizedBox.shrink(); + } + return WarningBanner(message: message); + } + + /// A GPU feature's prerequisites (RIFE, AI upscaling), the moment it is + /// chosen. Unlike the OpenCL options there is no fallback — such a job on a + /// machine without a Vulkan device is refused — so this says exactly what + /// is missing and how to fix it (the worker's wording, see `gpu_plugin.rs`), + /// with a button to check again once the user has. + Widget _buildGpuPluginNotice( BuildContext context, MainViewModel viewModel, String filterId, DynamicParameters params, ) { - if (filterId != 'frame_rate') return const SizedBox.shrink(); - final notice = rifeNoticeFor( - method: params.values['method'], - status: viewModel.rifeStatus, - checking: viewModel.rifeChecking, + final plugin = gpuPluginSelectedBy(filterId, params.values); + if (plugin == null) return const SizedBox.shrink(); + final status = viewModel.gpuPluginStatus(plugin); + final notice = gpuPluginNoticeFor( + selected: true, + status: status, + checking: viewModel.gpuPluginChecking(plugin), ); switch (notice) { - case RifeNotice.none: + case GpuPluginNotice.none: return const SizedBox.shrink(); - case RifeNotice.checking: + case GpuPluginNotice.checking: return Padding( padding: const EdgeInsets.only(bottom: 12), child: Row( @@ -193,19 +288,19 @@ class PassSettingsInline extends StatelessWidget { const SizedBox(width: 8), Expanded( child: Text( - 'Checking whether RIFE can run on this computer…', + 'Checking whether ${plugin.label} can run on this computer…', style: Theme.of(context).textTheme.bodySmall, ), ), ], ), ); - case RifeNotice.unavailable: + case GpuPluginNotice.unavailable: final scheme = Theme.of(context).colorScheme; return WarningBanner( - message: viewModel.rifeStatus!.message!, + message: status!.message!, action: OutlinedButton.icon( - onPressed: viewModel.recheckRife, + onPressed: () => viewModel.recheckGpuPlugin(plugin), icon: const Icon(Icons.refresh, size: 16), label: const Text('Check again'), style: OutlinedButton.styleFrom( diff --git a/app/test/attribution_test.dart b/app/test/attribution_test.dart index b4fd511a..809cd0cf 100644 --- a/app/test/attribution_test.dart +++ b/app/test/attribution_test.dart @@ -58,6 +58,7 @@ const _pluginToNotice = { 'dctfilter': 'DCTFilter', 'deblock': 'Deblock', 'dedot': 'DeDot', + 'descale': 'descale — MIT', 'descratch': 'DeScratch', 'dfttest': 'DFTTest', 'eedi3m': 'EEDI3', @@ -159,21 +160,31 @@ void main() { }); test('credits every file the bundle carries outside the plugin directory', () { - // RIFE lives in vapoursynth/rife/, outside the autoload directory, so it - // is listed under _extraFiles rather than in the per-platform lists. + // The ncnn plugins live outside the autoload directory (RIFE in + // vapoursynth/rife/, VSNCNN and its models in vapoursynth/ncnn/, MoltenVK + // in vapoursynth/moltenvk/), so they are listed under _extraFiles rather + // than in the per-platform lists. final manifest = jsonDecode( File(p.join(root, 'Scripts', 'deps-expected-plugins.json')).readAsStringSync(), ) as Map; final extra = (manifest['_extraFiles'] as Map); + // Matched by prefix of the file's stem: the model files share one per family. const fileToNotice = { 'librife': 'RIFE (VapourSynth-RIFE-ncnn-Vulkan)', 'libmoltenvk': 'MoltenVK', 'flownet': 'RIFE 4.26 model', + 'libvsncnn': 'VSNCNN (vs-mlrt)', + 'vsncnn': 'VSNCNN (vs-mlrt)', + 'realesrganv2-animevideo': 'Real-ESRGAN anime video model', + 'up2x-latest': 'Real-CUGAN models', }; for (final entry in extra.entries) { for (final path in (entry.value as List).cast()) { final stem = p.basenameWithoutExtension(path).toLowerCase(); - final needle = fileToNotice[stem]; + final key = fileToNotice.keys + .where((k) => stem == k || stem.startsWith('$k-')) + .firstOrNull; + final needle = key == null ? null : fileToNotice[key]; expect(needle, isNotNull, reason: '${entry.key}: $path has no entry in fileToNotice; credit it ' 'in NOTICES.txt from its upstream LICENSE, then map it here.'); @@ -183,36 +194,46 @@ void main() { } }); - test('RIFE ships exactly where the worker says it does', () { - // The worker reports "not available for this platform" from - // rife::platform_ships_rife; a bundle that ships RIFE where the worker - // says it can't (or the reverse) is a feature nobody can reach, or a - // probe that blames an "older bundle" that is in fact current. - final manifest = jsonDecode( - File(p.join(root, 'Scripts', 'deps-expected-plugins.json')).readAsStringSync(), - ) as Map; - final bundles = (manifest['_extraFiles'] as Map) - .entries - .where((e) => (e.value as List).any((f) => f.toString().contains('/rife/'))) - .map((e) => e.key.replaceAll(RegExp(r'-v2$'), '')) - .toSet(); - final source = - File(p.join(root, 'worker', 'src', 'rife.rs')).readAsStringSync(); - final match = RegExp(r'fn platform_ships_rife[^{]*\{\s*matches!\(platform_suffix,([^)]*)\)') - .firstMatch(source); - expect(match, isNotNull, reason: 'could not parse platform_ships_rife'); - final worker = RegExp(r'"([a-z0-9-]+)"') - .allMatches(match!.group(1)!) - .map((m) => m.group(1)!) - .toSet(); - expect(bundles, worker); - // And both tiers of an x64 platform carry it, or v2 users lose it. - for (final platform in worker.where((p) => p.endsWith('-x64'))) { - if (platform.startsWith('macos')) continue; - expect(manifest['_extraFiles']['$platform-v2'], manifest['_extraFiles'][platform], - reason: '$platform-v2 must carry the same RIFE files as $platform'); - } - }); + // The worker reports "not available for this platform" from + // gpu_plugin::platform_ships_*; a bundle that ships a plugin where the + // worker says it can't (or the reverse) is a feature nobody can reach, or a + // probe that blames an "older bundle" that is in fact current. + for (final (name, dir, fn) in [ + ('RIFE', '/rife/', 'platform_ships_rife'), + ('AI upscaling', '/ncnn/', 'platform_ships_upscale'), + ]) { + test('$name ships exactly where the worker says it does', () { + final manifest = jsonDecode( + File(p.join(root, 'Scripts', 'deps-expected-plugins.json')).readAsStringSync(), + ) as Map; + final bundles = (manifest['_extraFiles'] as Map) + .entries + .where((e) => (e.value as List).any((f) => f.toString().contains(dir))) + .map((e) => e.key.replaceAll(RegExp(r'-v2$'), '')) + .toSet(); + final source = + File(p.join(root, 'worker', 'src', 'gpu_plugin.rs')).readAsStringSync(); + final match = RegExp('fn $fn[^{]*\\{\\s*matches!\\(platform_suffix,([^)]*)\\)') + .firstMatch(source); + expect(match, isNotNull, reason: 'could not parse $fn'); + final worker = RegExp(r'"([a-z0-9-]+)"') + .allMatches(match!.group(1)!) + .map((m) => m.group(1)!) + .toSet(); + expect(bundles, worker); + // And both tiers of an x64 platform carry it, or v2 users lose it. + for (final platform in worker.where((p) => p.endsWith('-x64'))) { + if (platform.startsWith('macos')) continue; + expect(manifest['_extraFiles']['$platform-v2'], manifest['_extraFiles'][platform], + reason: '$platform-v2 must carry the same files as $platform'); + } + // macOS resolves Vulkan from the shared MoltenVK. + if (worker.contains('macos-arm64')) { + expect(manifest['_extraFiles']['macos-arm64'], + contains('vapoursynth/moltenvk/libMoltenVK.dylib')); + } + }); + } test('does not credit components that are no longer shipped', () { for (final gone in _mustNotAppear) { diff --git a/app/test/descale_detect_widget_test.dart b/app/test/descale_detect_widget_test.dart new file mode 100644 index 00000000..8dd327f8 --- /dev/null +++ b/app/test/descale_detect_widget_test.dart @@ -0,0 +1,112 @@ +// Descale's Detect: the panel runs the analysis on the selected video and +// offers what it found — and when a video has no clear native resolution, it +// says so instead of offering a guess, because a wrong one damages the picture. + +import 'dart:convert'; +import 'dart:io'; + +import 'package:flutter/material.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:provider/provider.dart'; +import 'package:shared_preferences/shared_preferences.dart'; +import 'package:vapourbox/models/filter_registry.dart'; +import 'package:vapourbox/models/native_res_report.dart'; +import 'package:vapourbox/models/processing_pipeline.dart'; +import 'package:vapourbox/models/video_job.dart'; +import 'package:vapourbox/services/advanced_mode_service.dart'; +import 'package:vapourbox/viewmodels/main_viewmodel.dart'; +import 'package:vapourbox/views/pass_settings/pass_settings_inline.dart'; + +import 'fakes/fake_job_runner.dart'; + +void main() { + TestWidgetsFlutterBinding.ensureInitialized(); + + late MainViewModel vm; + late List asked; + late NativeResReport answer; + + setUp(() { + SharedPreferences.setMockInitialValues({}); + AdvancedModeService.instance.resetForTesting(); + FilterRegistry.instance.registerFromJson( + jsonDecode(File('assets/filters/core/descale.json').readAsStringSync()) + as Map, + ); + asked = []; + vm = MainViewModel( + runner: FakeJobRunner(), + nativeResDetector: (job) async { + asked.add(job); + return answer; + }, + ); + }); + + tearDown(() => vm.dispose()); + + Future pumpPanel(WidgetTester tester) async { + await tester.pumpWidget( + ChangeNotifierProvider.value( + value: AdvancedModeService.instance, + child: ChangeNotifierProvider.value( + value: vm, + child: const MaterialApp( + home: Scaffold( + body: SingleChildScrollView( + child: PassSettingsInline(passType: PassType.descale), + ), + ), + ), + ), + ), + ); + await tester.pump(); + } + + Future settle(WidgetTester tester) => tester.pump(const Duration(seconds: 1)); + + testWidgets('without a selected video, Detect cannot run', (tester) async { + await pumpPanel(tester); + final button = tester.widget(find.widgetWithText(OutlinedButton, 'Detect native resolution')); + expect(button.onPressed, isNull); + expect(find.text('Select a video to analyse.'), findsOneWidget); + await settle(tester); + }); + + testWidgets('a clear match is offered, and Use these settings applies it', (tester) async { + answer = const NativeResReport( + found: true, height: 810, width: 1440, kernel: 'bicubic', + bicubicB: 1 / 3, bicubicC: 1 / 3, dip: 0.12, + kernels: [('Bicubic (Mitchell)', 1e-5), ('Bilinear', 4e-5)], + ); + vm.debugAddQueueItemForTest('/videos/episode1.mkv', width: 1920, height: 1080); + await pumpPanel(tester); + await tester.tap(find.text('Detect native resolution')); + await tester.pump(); + await tester.pump(); + expect(asked.single.inputPath, '/videos/episode1.mkv'); + expect(asked.single.inputHeight, 1080); + expect(find.text('Drawn at 1440×810, upscaled with Bicubic (Mitchell).'), findsOneWidget); + + await tester.tap(find.text('Use these settings')); + await tester.pump(); + final ds = vm.processingPipeline.descale; + expect(ds.enabled, isTrue); + expect(ds.nativeHeight, 810); + expect(ds.bicubicB, closeTo(1 / 3, 1e-9)); + await settle(tester); + }); + + testWidgets('no clear native resolution: said plainly, nothing to apply', (tester) async { + answer = const NativeResReport(found: false, height: 559, width: 699, kernel: 'bicubic', dip: 0.91); + vm.debugAddQueueItemForTest('/videos/vhs.avi', width: 720, height: 576); + await pumpPanel(tester); + await tester.tap(find.text('Detect native resolution')); + await tester.pump(); + await tester.pump(); + expect(find.textContaining('No clear native resolution'), findsOneWidget); + expect(find.text('Use these settings'), findsNothing); + await settle(tester); + }); +} diff --git a/app/test/filter_schema_curation_test.dart b/app/test/filter_schema_curation_test.dart index fe338462..0c30c717 100644 --- a/app/test/filter_schema_curation_test.dart +++ b/app/test/filter_schema_curation_test.dart @@ -174,6 +174,10 @@ void main() { 'fft3dfilter', 'ttempsmooth', 'temporal_degrain2', + // BM3D and KNLMeans (1.5.0) are advanced too: the simple slots are full, + // and KNLMeans needs an OpenCL device most users can't reason about. + 'bm3d', + 'knlmeans', ]) { expect(nr.getMethod(id)?.advancedOnly, true, reason: id); } diff --git a/app/test/gpu_plugin_notice_widget_test.dart b/app/test/gpu_plugin_notice_widget_test.dart new file mode 100644 index 00000000..db5514ac --- /dev/null +++ b/app/test/gpu_plugin_notice_widget_test.dart @@ -0,0 +1,124 @@ +// The panels tell the user, the moment a GPU feature is chosen — RIFE on Frame +// Rate, Real-ESRGAN / Real-CUGAN on Crop & Resize — why it can't run here and +// what to do: the worker's message, verbatim, with a way to check again once +// they have. Neither has a fallback (such a job is refused), so this notice is +// the user's only warning before a job fails. + +import 'dart:convert'; +import 'dart:io'; + +import 'package:flutter/material.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:provider/provider.dart'; +import 'package:shared_preferences/shared_preferences.dart'; +import 'package:vapourbox/models/dynamic_parameters.dart'; +import 'package:vapourbox/models/filter_registry.dart'; +import 'package:vapourbox/models/gpu_plugin_status.dart'; +import 'package:vapourbox/models/processing_pipeline.dart'; +import 'package:vapourbox/services/advanced_mode_service.dart'; +import 'package:vapourbox/viewmodels/main_viewmodel.dart'; +import 'package:vapourbox/views/pass_settings/pass_settings_inline.dart'; + +import 'fakes/fake_job_runner.dart'; + +void main() { + TestWidgetsFlutterBinding.ensureInitialized(); + + late MainViewModel vm; + + setUp(() { + SharedPreferences.setMockInitialValues({}); + AdvancedModeService.instance.resetForTesting(); + for (final id in ['frame_rate', 'crop_resize']) { + FilterRegistry.instance.registerFromJson( + jsonDecode(File('assets/filters/core/$id.json').readAsStringSync()) + as Map, + ); + } + vm = MainViewModel(runner: FakeJobRunner()); + }); + + tearDown(() => vm.dispose()); + + Future pumpPanel(WidgetTester tester, PassType pass) async { + await tester.pumpWidget( + ChangeNotifierProvider.value( + value: AdvancedModeService.instance, + child: ChangeNotifierProvider.value( + value: vm, + child: MaterialApp( + home: Scaffold( + body: SingleChildScrollView(child: PassSettingsInline(passType: pass)), + ), + ), + ), + ), + ); + await tester.pump(); + } + + /// Lets the view model's preview debounce (300 ms) fire before teardown. + Future settle(WidgetTester tester) => tester.pump(const Duration(seconds: 1)); + + // Each feature: its pass, how to select it, and how to select something else. + final cases = { + 'RIFE': ( + PassType.frameRate, + GpuPlugin.rife, + () => vm.updateDynamicParams('frame_rate', const DynamicParameters( + filterId: 'frame_rate', enabled: true, + values: {'method': 'rife', 'target': 'pal50', 'blockSize': 16, 'overlap': 8})), + () => vm.updateDynamicParams('frame_rate', const DynamicParameters( + filterId: 'frame_rate', enabled: true, + values: {'method': 'flowFps', 'target': 'pal50', 'blockSize': 16, 'overlap': 8})), + ), + 'AI upscaling': ( + PassType.cropResize, + GpuPlugin.upscale, + () => vm.updateDynamicParams('crop_resize', const DynamicParameters( + filterId: 'crop_resize', enabled: true, + values: {'useIntegerUpscale': true, 'upscaleMethod': 'realCugan', 'upscaleFactor': 2})), + () => vm.updateDynamicParams('crop_resize', const DynamicParameters( + filterId: 'crop_resize', enabled: true, + values: {'useIntegerUpscale': true, 'upscaleMethod': 'nnedi3Rpow2', 'upscaleFactor': 2})), + ), + }; + + for (final MapEntry(key: name, value: (pass, plugin, select, selectOther)) in cases.entries) { + final message = '${plugin.label} needs a graphics card with Vulkan support, and none was found.'; + final unavailable = GpuPluginStatus(available: false, reason: 'noVulkan', message: message); + + testWidgets('$name selected where it cannot run: the remedy and Check again', (tester) async { + select(); + vm.debugSetGpuPluginStatus(plugin, unavailable); + await pumpPanel(tester, pass); + expect(find.text(message), findsOneWidget); + expect(find.text('Check again'), findsOneWidget); + await settle(tester); + }); + + testWidgets('$name: while the probe runs, it says so', (tester) async { + select(); + vm.debugSetGpuPluginStatus(plugin, null, checking: true); + await pumpPanel(tester, pass); + expect(find.textContaining('Checking whether ${plugin.label} can run'), findsOneWidget); + expect(find.text(message), findsNothing); + await settle(tester); + }); + + testWidgets('$name: silent where it runs, or when something else is chosen', (tester) async { + select(); + vm.debugSetGpuPluginStatus(plugin, const GpuPluginStatus(available: true)); + await pumpPanel(tester, pass); + expect(find.text('Check again'), findsNothing); + expect(find.textContaining('can run on this computer'), findsNothing); + + selectOther(); + vm.debugSetGpuPluginStatus(plugin, unavailable); + await pumpPanel(tester, pass); + expect(find.text(message), findsNothing); + expect(find.text('Check again'), findsNothing); + await settle(tester); + }); + } +} diff --git a/app/test/gpu_plugin_status_test.dart b/app/test/gpu_plugin_status_test.dart new file mode 100644 index 00000000..fe41b9ce --- /dev/null +++ b/app/test/gpu_plugin_status_test.dart @@ -0,0 +1,125 @@ +// The GPU features' status as the app reads it from `--probe-rife` / +// `--probe-upscale`, and the panels' decision about what to say. Pure unit +// tests; the probes themselves run against the real worker in +// integration_gpu_plugins_test.dart. +import 'package:flutter_test/flutter_test.dart'; + +import 'package:vapourbox/models/crop_resize_parameters.dart'; +import 'package:vapourbox/models/dynamic_parameters.dart'; +import 'package:vapourbox/models/frame_rate_parameters.dart'; +import 'package:vapourbox/models/gpu_plugin_status.dart'; +import 'package:vapourbox/models/parameter_converter.dart'; + +void main() { + group('GpuPluginStatus.fromJson', () { + test('available', () { + final s = GpuPluginStatus.fromJson(GpuPlugin.rife, {'available': true}); + expect(s.available, isTrue); + expect(s.message, isNull); + }); + + test('unavailable carries the worker message verbatim', () { + final s = GpuPluginStatus.fromJson(GpuPlugin.rife, { + 'available': false, + 'reason': 'missingLibrary', + 'detail': 'could not load vcomp140.dll', + 'message': 'RIFE needs the Microsoft Visual C++ Redistributable (x64)…', + }); + expect(s.available, isFalse); + expect(s.reason, 'missingLibrary'); + expect(s.message, startsWith('RIFE needs the Microsoft Visual C++')); + }); + + test('an unavailable answer without a message is never shown as blank', () { + for (final plugin in GpuPlugin.values) { + final s = GpuPluginStatus.fromJson(plugin, {'available': false, 'reason': 'noVulkan'}); + expect(s.available, isFalse); + expect(s.message, contains(plugin.label)); + } + }); + + test('each plugin has its own probe flag', () { + expect(GpuPlugin.rife.probeFlag, '--probe-rife'); + expect(GpuPlugin.upscale.probeFlag, '--probe-upscale'); + }); + }); + + group('gpuPluginSelectedBy', () { + test('Frame Rate selects RIFE only for the rife method', () { + expect(gpuPluginSelectedBy('frame_rate', {'method': 'rife'}), GpuPlugin.rife); + for (final m in ['flowFps', 'duplicate', null]) { + expect(gpuPluginSelectedBy('frame_rate', {'method': m}), isNull); + } + }); + + test('Crop & Resize selects AI upscaling only for an AI method with upscaling on', () { + for (final m in ['realEsrgan', 'realCugan']) { + expect(gpuPluginSelectedBy('crop_resize', {'useIntegerUpscale': true, 'upscaleMethod': m}), + GpuPlugin.upscale); + expect(gpuPluginSelectedBy('crop_resize', {'useIntegerUpscale': false, 'upscaleMethod': m}), + isNull); + } + for (final m in ['nnedi3Rpow2', 'eedi3Rpow2', 'spline36']) { + expect(gpuPluginSelectedBy('crop_resize', {'useIntegerUpscale': true, 'upscaleMethod': m}), + isNull); + } + // Every AI method in the enum is recognised. + for (final m in UpscaleMethod.values.where((m) => m.name.startsWith('real'))) { + expect(gpuPluginSelectedBy('crop_resize', {'useIntegerUpscale': true, 'upscaleMethod': m.name}), + GpuPlugin.upscale, reason: m.name); + } + }); + + test('other passes select nothing', () { + expect(gpuPluginSelectedBy('noise_reduction', {'method': 'rife'}), isNull); + }); + }); + + group('gpuPluginNoticeFor', () { + const ok = GpuPluginStatus(available: true); + const no = GpuPluginStatus(available: false, reason: 'noVulkan', message: 'no GPU'); + + test('says nothing unless the feature is selected', () { + for (final status in [ok, no, null]) { + expect(gpuPluginNoticeFor(selected: false, status: status, checking: false), + GpuPluginNotice.none); + } + }); + + test('selected: checking, then unavailable or silent', () { + expect(gpuPluginNoticeFor(selected: true, status: null, checking: false), + GpuPluginNotice.checking); + expect(gpuPluginNoticeFor(selected: true, status: no, checking: true), + GpuPluginNotice.checking); + expect(gpuPluginNoticeFor(selected: true, status: no, checking: false), + GpuPluginNotice.unavailable); + expect(gpuPluginNoticeFor(selected: true, status: ok, checking: false), + GpuPluginNotice.none); + }); + }); + + test('the converter round-trips every Frame Rate method', () { + for (final method in FrameRateMethod.values) { + final params = FrameRateParameters(enabled: true, method: method); + final dynamic = ParameterConverter.fromFrameRate(params); + expect(dynamic.values['method'], method.name); + expect(ParameterConverter.toFrameRate(dynamic).method, method); + } + // An unknown id from an old or hand-edited preset falls back to FlowFPS. + final unknown = const DynamicParameters( + filterId: 'frame_rate', enabled: true, values: {'method': 'bogus'}); + expect(ParameterConverter.toFrameRate(unknown).method, FrameRateMethod.flowFps); + }); + + test('the converter round-trips every upscale method and CUGAN strength', () { + for (final method in UpscaleMethod.values) { + for (final noise in ['none', 'conservative', 'denoise3x']) { + final params = CropResizeParameters( + enabled: true, useIntegerUpscale: true, upscaleMethod: method, cuganNoise: noise); + final back = ParameterConverter.toCropResize(ParameterConverter.fromCropResize(params)); + expect(back.upscaleMethod, method); + expect(back.cuganNoise, noise); + } + } + }); +} diff --git a/app/test/integration_ai_upscale_test.dart b/app/test/integration_ai_upscale_test.dart new file mode 100644 index 00000000..6c447113 --- /dev/null +++ b/app/test/integration_ai_upscale_test.dart @@ -0,0 +1,348 @@ +// Real-ESRGAN / Real-CUGAN upscaling, encoded end to end and checked by pixels. +// +// Wherever AI upscaling ships, CI gives the runner a Vulkan device (a software +// one, lavapipe, on the GPU-less Linux and Windows runners) and sets +// VAPOURBOX_EXPECT_VULKAN=1, so these must run there; see +// support/gpu_expectations.dart. The source is the small progressive clip +// (320x240) and each job encodes five frames, so even 4x CUGAN on a CPU Vulkan +// driver finishes in minutes. It must be progressive: on a combed source the +// networks rightly remove the combing, and that difference from Spline36 — +// real detail at the source's own resolution — would swamp the comparison. +// +// What is checked, per method and factor: +// * size and frame count — an upscale changes the size, never the count; +// * on real footage, the network ran (the result is not plain resampling) +// and stays close to a Spline36 upscale of the same frames — a loose sanity +// bound only: Real-ESRGAN 2x legitimately redraws ~5 levels' worth, and +// this clip has duplicate frames, so it cannot prove colour or position; +// * on a synthetic pattern, which can: a white bar's centre stays where +// Spline36 puts it (within 0.75 of an output pixel, both axes) for every +// model — the networks widen a hard edge by up to ~1.5 px each side, which +// moves either edge on its own but not the centre — and flat colour +// patches keep their colour. Real-CUGAN 2x reproduces flat colour to under +// a level, and every method shares the template's conversions, so it is +// the strict check of the matrix and range (a mix-up moves these patches +// 5-20 levels). Real-ESRGAN measurably tints and saturates flat colour +// (measured up to 3 levels at 2x and 11 at 4x, in fp32 too, with vs-mlrt's +// own wrapper doing nothing more than this template — the model, not the +// pipeline) and CUGAN 4x drifts ~4, so those get bounds just above that; +// * the preview of a frame is the encode's frame (the same script runs); +// * a 10-bit source stays 10-bit and matches the 8-bit result. +@Tags(['heavy']) +library; + +import 'dart:io'; +import 'dart:typed_data'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:path/path.dart' as p; +import 'package:uuid/uuid.dart'; + +import 'package:vapourbox/models/crop_resize_parameters.dart'; +import 'package:vapourbox/models/encoding_settings.dart'; +import 'package:vapourbox/models/processing_pipeline.dart'; +import 'package:vapourbox/models/qtgmc_parameters.dart'; +import 'package:vapourbox/models/video_job.dart'; + +import 'support/gpu_expectations.dart'; +import 'support/worker_harness.dart'; + +const _start = 100; +const _w = 320, _h = 240; +String get _input => p.join(WorkerHarness.repoRoot, 'Tests', 'TestResources', 'small_clip.mp4'); +const _frames = 5; +String get _outDir => p.join(WorkerHarness.outputDir, 'ai_upscale'); + +VideoJob _job(String name, UpscaleMethod method, int factor, + {String noise = 'conservative', String? input, String pixFmt = 'yuv420p'}) => + VideoJob( + id: const Uuid().v4(), + inputPath: input ?? _input, + outputPath: p.join(_outDir, '$name.mkv'), + processingPipeline: ProcessingPipeline( + deinterlace: const QTGMCParameters(enabled: false), + cropResize: CropResizeParameters( + enabled: true, + useIntegerUpscale: true, + upscaleMethod: method, + upscaleFactor: factor, + cuganNoise: noise, + ), + ), + encodingSettings: const EncodingSettings( + codec: VideoCodec.ffv1, + container: ContainerFormat.mkv, + audioMode: AudioMode.none, + ), + inputWidth: _w, + inputHeight: _h, + inputPixelFormat: pixFmt, + inputFrameRate: 29.97, + // The post-trim count: the decoder pipes exactly these frames. + startFrame: _start, + endFrame: _start + _frames - 1, + totalFrames: _frames, + ); + +Future _encode(VideoJob job, String label) async { + final r = await WorkerHarness.runJob(job.toJson(), label: label, + timeout: const Duration(minutes: 20)); + expect(r.success, isTrue, reason: '$label: ${r.error}\n${r.logs.join('\n')}'); + return r.outputPath!; +} + +/// Frame [index] of [path], area-downscaled to the source size. +Future _atSourceSize(String path, int index, String label) async { + final png = File(p.join(Directory.systemTemp.path, + 'vb_${label}_${DateTime.now().microsecondsSinceEpoch}.png')); + try { + final r = await Process.run( + WorkerHarness.ffmpegPath, + ['-y', '-v', 'error', '-i', path, + '-vf', 'select=eq(n\\,$index),scale=$_w:$_h:flags=area', '-frames:v', '1', png.path], + environment: WorkerHarness.ffmpegEnv, + ); + expect(r.exitCode, 0, reason: '${r.stderr}'); + return WorkerHarness.imageToRgb24(await png.readAsBytes(), label: label); + } finally { + if (await png.exists()) await png.delete(); + } +} + +Future> _stream(String path) async => + (await WorkerHarness.firstStream(path, + selector: 'v:0', + entries: ['width', 'height', 'pix_fmt', 'nb_read_frames'], + countFrames: true))!; + +void main() { + // One Spline36 reference per factor, shared by every method at that factor. + final references = >{}; + Future reference(int factor) => references.putIfAbsent( + factor, + () => _encode(_job('spline36_${factor}x', UpscaleMethod.spline36, factor), + 'spline36_${factor}x')); + + setUpAll(() async { + await WorkerHarness.ensureReady(); + await Directory(_outDir).create(recursive: true); + // ignore: avoid_print + print('AI upscaling on $platformId (expect Vulkan: $expectVulkan)'); + }); + + final cases = <(UpscaleMethod, int, String)>[ + (UpscaleMethod.realEsrgan, 2, 'conservative'), + (UpscaleMethod.realEsrgan, 4, 'conservative'), + (UpscaleMethod.realCugan, 2, 'none'), + (UpscaleMethod.realCugan, 2, 'conservative'), + (UpscaleMethod.realCugan, 2, 'denoise3x'), + (UpscaleMethod.realCugan, 4, 'conservative'), + ]; + + for (final (method, factor, noise) in cases) { + final name = '${method.name}_${factor}x${method == UpscaleMethod.realCugan ? '_$noise' : ''}'; + test(name, () async { + if (!await requireGpuPlugin('--probe-upscale')) return; + final job = _job(name, method, factor, noise: noise); + final out = await _encode(job, name); + + final s = await _stream(out); + expect(s['width'], _w * factor); + expect(s['height'], _h * factor); + expect(s['pix_fmt'], 'yuv420p'); + expect(int.parse('${s['nb_read_frames']}'), _frames, + reason: 'an upscale must not change the frame count'); + + // The picture is the source's, and the network changed it. + final ref = await reference(factor); + for (final i in [0, 2, 4]) { + final small = WorkerHarness.meanAbsDiff( + await _atSourceSize(out, i, '${name}_s$i'), + await _atSourceSize(ref, i, 'ref${factor}_s$i')); + final full = WorkerHarness.meanAbsDiff( + await WorkerHarness.frameRgb24(out, i, label: '${name}_$i'), + await WorkerHarness.frameRgb24(ref, i, label: 'ref${factor}_$i')); + // ignore: avoid_print + print(' $name frame $i vs Spline36: at source size ${small.toStringAsFixed(2)}, ' + 'full size ${full.toStringAsFixed(2)}'); + expect(small, lessThan(8.0), reason: 'frame $i is not the same picture as the source'); + expect(full, greaterThan(0.5), reason: 'frame $i is plain resampling: no network ran'); + } + + // The preview runs the same script on the same frame. + final pv = await WorkerHarness.runPreview(job.toJson(), + frame: _start + 2, label: '${name}_pv'); + expect(pv.success, isTrue, reason: pv.errorTail); + final pvDiff = WorkerHarness.meanAbsDiff( + await WorkerHarness.imageToRgb24(pv.png!, label: '${name}_pv'), + await WorkerHarness.frameRgb24(out, 2, label: '${name}_enc2')); + // ignore: avoid_print + print(' $name preview vs encode: meanAbsDiff ${pvDiff.toStringAsFixed(2)}'); + expect(pvDiff, lessThan(1.0), reason: 'the preview is not the encoded picture'); + }, timeout: const Timeout(Duration(minutes: 30))); + } + + // Flat patches keep their colour and a hard edge keeps its position — the + // two things a wrong matrix, range or offset would break. + group('synthetic pattern', () { + late String pattern; + // Mid-saturation colours: nothing clips, so a range mix-up (which moves a + // channel ~10 levels here) or a BT.601/709 mix-up (5-20) cannot hide at + // 0 or 255, where the networks also roll off by a few levels. + const patches = { + '0xB04030': (16, 16), '0x30A050': (96, 16), '0x4060C0': (176, 16), + '0xC0B040': (16, 120), '0x40B0B0': (96, 120), '0xA040A0': (176, 120), + }; + const patch = 64, barX = 256; + + setUpAll(() async { + pattern = p.join(_outDir, 'pattern.mkv'); + final boxes = [ + for (final MapEntry(key: c, value: (x, y)) in patches.entries) + 'drawbox=x=$x:y=$y:w=$patch:h=$patch:color=$c@1:t=fill', + 'drawbox=x=$barX:y=40:w=48:h=160:color=white@1:t=fill', + ].join(','); + final r = await Process.run( + WorkerHarness.ffmpegPath, + ['-y', '-v', 'error', '-f', 'lavfi', + '-i', 'color=c=0x808080:s=${_w}x$_h:r=25:d=0.4,format=yuv420p,$boxes', + '-c:v', 'ffv1', pattern], + environment: WorkerHarness.ffmpegEnv, + ); + expect(r.exitCode, 0, reason: '${r.stderr}'); + }); + + VideoJob patternJob(String name, UpscaleMethod method, int factor) { + final j = _job(name, method, factor, input: pattern); + return VideoJob.fromJson({ + ...j.toJson(), + 'inputFrameRate': 25.0, + 'startFrame': 2, + 'endFrame': 4, + 'totalFrames': 3, + }); + } + + /// Mean RGB over a square region (output pixels), inset to avoid edges. + List meanRgb(Uint8List rgb, int width, int x0, int y0, int size) { + final sums = [0.0, 0.0, 0.0]; + var n = 0; + for (var y = y0; y < y0 + size; y++) { + for (var x = x0; x < x0 + size; x++) { + final i = (y * width + x) * 3; + for (var c = 0; c < 3; c++) { + sums[c] += rgb[i + c]; + } + n++; + } + } + return [for (final v in sums) v / n]; + } + + /// Where a line of luminance samples crosses half-way between its two + /// ends, to sub-pixel precision: an edge's position. + double crossing(double Function(int) lum, int from, int to) { + final mid = (lum(from) + lum(to)) / 2; + final rising = lum(to) > lum(from); + for (var i = from; i < to; i++) { + final a = lum(i), b = lum(i + 1); + if (rising ? (a < mid && b >= mid) : (a >= mid && b < mid)) { + return i + (mid - a) / (b - a); + } + } + return double.nan; + } + + /// The white bar's (left, right, top, bottom) edges in an output frame. + (double, double, double, double) barEdges(Uint8List rgb, int factor) { + final w = _w * factor; + double at(int x, int y) { + final i = (y * w + x) * 3; + return (rgb[i] + rgb[i + 1] + rgb[i + 2]) / 3; + } + final row = 120 * factor, col = (barX + 24) * factor, r = 16 * factor; + return ( + crossing((x) => at(x, row), barX * factor - r, barX * factor + r), + crossing((x) => at(x, row), (barX + 48) * factor - r, (barX + 48) * factor + r), + crossing((y) => at(col, y), 40 * factor - r, 40 * factor + r), + crossing((y) => at(col, y), 200 * factor - r, 200 * factor + r), + ); + } + + for (final (method, factor, colourTolerance) in [ + (UpscaleMethod.realEsrgan, 2, 6.0), (UpscaleMethod.realEsrgan, 4, 14.0), + (UpscaleMethod.realCugan, 2, 2.0), (UpscaleMethod.realCugan, 4, 6.0), + ]) { + test('${method.name} ${factor}x keeps colour and position', () async { + if (!await requireGpuPlugin('--probe-upscale')) return; + final name = 'pattern_${method.name}_${factor}x'; + final out = await _encode(patternJob(name, method, factor), name); + final ref = await _encode( + patternJob('pattern_spline36_${factor}x', UpscaleMethod.spline36, factor), + 'pattern_spline36_${factor}x'); + final w = _w * factor; + final a = await WorkerHarness.frameRgb24(out, 1, label: '${name}_1'); + final b = await WorkerHarness.frameRgb24(ref, 1, label: 'pref${factor}_1'); + + final inset = 12 * factor, size = (patch * factor) - 2 * inset; + for (final MapEntry(key: c, value: (x, y)) in patches.entries) { + final ma = meanRgb(a, w, x * factor + inset, y * factor + inset, size); + final mb = meanRgb(b, w, x * factor + inset, y * factor + inset, size); + final worst = [for (var i = 0; i < 3; i++) (ma[i] - mb[i]).abs()] + .reduce((p, q) => p > q ? p : q); + // ignore: avoid_print + print(' $name $c: ${ma.map((v) => v.toStringAsFixed(1)).toList()} ' + 'vs ${mb.map((v) => v.toStringAsFixed(1)).toList()}'); + expect(worst, lessThan(colourTolerance), reason: '$c patch changed colour'); + } + + final (l, rt, t, bt) = barEdges(a, factor); + final (rl, rr, rtop, rb) = barEdges(b, factor); + final cx = (l + rt) / 2, cy = (t + bt) / 2; + final rcx = (rl + rr) / 2, rcy = (rtop + rb) / 2; + // ignore: avoid_print + print(' $name bar centre (${cx.toStringAsFixed(2)}, ${cy.toStringAsFixed(2)}) ' + 'width ${(rt - l).toStringAsFixed(2)}; Spline36 (${rcx.toStringAsFixed(2)}, ' + '${rcy.toStringAsFixed(2)}) width ${(rr - rl).toStringAsFixed(2)}'); + expect((cx - rcx).abs(), lessThan(0.75), reason: 'the picture moved horizontally'); + expect((cy - rcy).abs(), lessThan(0.75), reason: 'the picture moved vertically'); + // Spline36 itself lands on factor x source: the reference is right. + expect((rcx - ((barX + 24) * factor - 0.5)).abs(), lessThan(0.25)); + expect(((rt - l) - (rr - rl)).abs(), lessThan(4.0), reason: 'the bar changed size'); + }, timeout: const Timeout(Duration(minutes: 30))); + } + }); + + test('a 10-bit source stays 10-bit and matches the 8-bit result', () async { + if (!await requireGpuPlugin('--probe-upscale')) return; + // Both fixtures from the same conversion, so frame N is the same picture + // in each (the AVI's own frame count differs between decoders). + final sources = {}; + for (final fmt in ['yuv420p', 'yuv420p10le']) { + final path = p.join(_outDir, 'source_$fmt.mkv'); + final made = await Process.run( + WorkerHarness.ffmpegPath, + ['-y', '-v', 'error', '-i', _input, '-frames:v', '200', '-pix_fmt', fmt, + '-c:v', 'ffv1', path], + environment: WorkerHarness.ffmpegEnv, + ); + expect(made.exitCode, 0, reason: '${made.stderr}'); + sources[fmt] = path; + } + + final eight = await _encode( + _job('esrgan_8bit', UpscaleMethod.realEsrgan, 2, input: sources['yuv420p']), + 'esrgan_8bit'); + final ten = await _encode( + _job('esrgan_10bit', UpscaleMethod.realEsrgan, 2, + input: sources['yuv420p10le'], pixFmt: 'yuv420p10le'), + 'esrgan_10bit'); + expect((await _stream(ten))['pix_fmt'], 'yuv420p10le'); + final Uint8List a = await WorkerHarness.frameRgb24(eight, 2, label: 'esrgan8'); + final Uint8List b = await WorkerHarness.frameRgb24(ten, 2, label: 'esrgan10'); + final diff = WorkerHarness.meanAbsDiff(a, b); + // ignore: avoid_print + print(' 8-bit vs 10-bit: meanAbsDiff ${diff.toStringAsFixed(2)}'); + expect(diff, lessThan(2.0), reason: 'the 10-bit path is not the same picture'); + }, timeout: const Timeout(Duration(minutes: 30))); +} diff --git a/app/test/integration_descale_test.dart b/app/test/integration_descale_test.dart new file mode 100644 index 00000000..3420ca3b --- /dev/null +++ b/app/test/integration_descale_test.dart @@ -0,0 +1,209 @@ +// Descale and its native-resolution detection, end to end on fixtures with a +// known answer: testsrc2 drawn at 1280x720 and upscaled to 1080p by ffmpeg with +// a chosen kernel, the way a studio master is made. The grid block stands in +// for credits drawn at full resolution on top of the upscale. +// +// * Detect finds 720 and the kernel actually used (Catmull-Rom, bilinear), +// and finds nothing on a source that was never upscaled (the VHS capture); +// * the encode keeps the size and frame count, stays close to the source, +// and with "Protect credits and text" leaves the grid exactly as it was — +// without it, the grid is smeared (the mask is what protects it); +// * the preview is the encode's frame. +@Tags(['heavy']) +library; + +import 'dart:convert'; +import 'dart:io'; +import 'dart:typed_data'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:path/path.dart' as p; +import 'package:uuid/uuid.dart'; + +import 'package:vapourbox/models/descale_parameters.dart'; +import 'package:vapourbox/models/encoding_settings.dart'; +import 'package:vapourbox/models/processing_pipeline.dart'; +import 'package:vapourbox/models/qtgmc_parameters.dart'; +import 'package:vapourbox/models/video_job.dart'; + +import 'support/worker_harness.dart'; + +const _frames = 12; +String get _dir => p.join(WorkerHarness.outputDir, 'descale'); + +Future _master(String name, String scaleFlags, {bool credits = false}) async { + final out = p.join(_dir, '$name.mkv'); + final up = 'scale=1920:1080:flags=$scaleFlags,format=yuv420p'; + final args = credits + ? ['-filter_complex', + '[0:v]$up[up];color=c=black:s=320x120:r=30000/1001:d=0.4,format=yuv420p,' + 'drawgrid=w=4:h=4:t=1:c=white[g];[up][g]overlay=x=100:y=100:shortest=1[out]', + '-map', '[out]'] + : ['-vf', up]; + final r = await Process.run( + WorkerHarness.ffmpegPath, + ['-y', '-v', 'error', '-f', 'lavfi', '-i', 'testsrc2=s=1280x720:r=30000/1001:d=0.4', + ...args, '-c:v', 'ffv1', out], + environment: WorkerHarness.ffmpegEnv, + ); + expect(r.exitCode, 0, reason: '${r.stderr}'); + return out; +} + +VideoJob _job(String name, String input, DescaleParameters ds, + {int width = 1920, int height = 1080, String pixFmt = 'yuv420p', double fps = 29.97, + int total = _frames}) => + VideoJob( + id: const Uuid().v4(), + inputPath: input, + outputPath: p.join(_dir, '$name.mkv'), + processingPipeline: ProcessingPipeline( + deinterlace: const QTGMCParameters(enabled: false), + descale: ds, + ), + encodingSettings: const EncodingSettings( + codec: VideoCodec.ffv1, container: ContainerFormat.mkv, audioMode: AudioMode.none), + inputWidth: width, + inputHeight: height, + inputPixelFormat: pixFmt, + inputFrameRate: fps, + totalFrames: total, + ); + +Future> _detect(VideoJob job) async { + final config = File(p.join(Directory.systemTemp.path, 'vb_detect_${job.id}.json')); + await config.writeAsString(jsonEncode(job.toJson())); + try { + final r = await Process.run( + WorkerHarness.workerPath, + ['--detect-native', '--config', config.path], + environment: WorkerHarness.workerEnv, + workingDirectory: File(WorkerHarness.workerPath).parent.path, + ); + final line = const LineSplitter().convert(r.stdout.toString()) + .lastWhere((l) => l.trim().startsWith('{'), orElse: () => ''); + expect(line, isNotEmpty, reason: 'no report: ${r.stdout}\n${r.stderr}'); + final report = jsonDecode(line) as Map; + // ignore: avoid_print + print(' detect ${p.basename(job.inputPath)}: found ${report['found']} ' + '${report['height']}p ${report['kernel']} b=${report['bicubicB']} ' + 'c=${report['bicubicC']} dip ${report['dip']}'); + expect(report['error'], isNull, reason: '${report['error']}'); + return report; + } finally { + await config.delete(); + } +} + +/// Frame [index]'s luma plane, 1920 wide. +Future _luma(String path, int index) async { + final out = File(p.join(Directory.systemTemp.path, + 'vb_luma_${DateTime.now().microsecondsSinceEpoch}.y')); + try { + final r = await Process.run( + WorkerHarness.ffmpegPath, + ['-y', '-v', 'error', '-i', path, '-vf', 'select=eq(n\\,$index),extractplanes=y', + '-frames:v', '1', '-f', 'rawvideo', out.path], + environment: WorkerHarness.ffmpegEnv, + ); + expect(r.exitCode, 0, reason: '${r.stderr}'); + return out.readAsBytes(); + } finally { + if (await out.exists()) await out.delete(); + } +} + +double _regionDiff(Uint8List a, Uint8List b, int x0, int y0, int x1, int y1) { + var sum = 0, n = 0; + for (var y = y0; y < y1; y++) { + for (var x = x0; x < x1; x++) { + sum += (a[y * 1920 + x] - b[y * 1920 + x]).abs(); + n++; + } + } + return sum / n; +} + +void main() { + late String catrom, bilinear, credits; + + setUpAll(() async { + await WorkerHarness.ensureReady(); + await Directory(_dir).create(recursive: true); + catrom = await _master('catrom', 'bicubic:param0=0:param1=0.5'); + bilinear = await _master('bilinear', 'bilinear'); + credits = await _master('credits', 'bicubic:param0=0:param1=0.5', credits: true); + }); + + group('Detect', () { + test('finds 720p Catmull-Rom on a Catmull-Rom master', () async { + final r = await _detect(_job('d_catrom', catrom, const DescaleParameters())); + expect(r['found'], isTrue); + expect((r['height'], r['width']), (720, 1280)); + expect(r['kernel'], 'bicubic'); + expect((r['bicubicB'] as num).toDouble(), closeTo(0, 1e-9)); + expect((r['bicubicC'] as num).toDouble(), closeTo(0.5, 1e-9)); + }, timeout: const Timeout(Duration(minutes: 10))); + + test('finds 720p bilinear on a bilinear master', () async { + final r = await _detect(_job('d_bilinear', bilinear, const DescaleParameters())); + expect(r['found'], isTrue); + expect(r['height'], 720); + expect(r['kernel'], 'bilinear'); + }, timeout: const Timeout(Duration(minutes: 10))); + + test('still finds it with credits drawn over the picture', () async { + final r = await _detect(_job('d_credits', credits, const DescaleParameters())); + expect(r['found'], isTrue); + expect(r['height'], 720); + }, timeout: const Timeout(Duration(minutes: 10))); + + test('finds nothing on a source that was never upscaled', () async { + final r = await _detect(_job('d_vhs', WorkerHarness.inputFile, const DescaleParameters(), + width: 720, height: 576, pixFmt: 'yuv422p', fps: 25, total: 75)); + expect(r['found'], isFalse, reason: 'a never-upscaled VHS capture reported a native resolution'); + }, timeout: const Timeout(Duration(minutes: 10))); + }); + + group('Encode', () { + const ds = DescaleParameters(enabled: true, nativeHeight: 720, kernel: DescaleKernel.bicubic, + bicubicB: 0, bicubicC: 0.5); + + test('keeps size and count, stays close, and protects the credits', () async { + final protected = await WorkerHarness.runJob(_job('e_protect', credits, ds).toJson(), + label: 'descale_protect'); + expect(protected.success, isTrue, reason: protected.error); + final s = (await WorkerHarness.firstStream(protected.outputPath!, + selector: 'v:0', entries: ['width', 'height', 'pix_fmt', 'nb_read_frames'], countFrames: true))!; + expect((s['width'], s['height'], s['pix_fmt']), (1920, 1080, 'yuv420p')); + expect(int.parse('${s['nb_read_frames']}'), _frames); + + final bare = await WorkerHarness.runJob( + _job('e_bare', credits, ds.copyWith(protectDetail: false)).toJson(), label: 'descale_bare'); + expect(bare.success, isTrue, reason: bare.error); + + final src = await _luma(credits, 3); + final withMask = await _luma(protected.outputPath!, 3); + final without = await _luma(bare.outputPath!, 3); + final gridKept = _regionDiff(withMask, src, 110, 110, 410, 210); + final gridLost = _regionDiff(without, src, 110, 110, 410, 210); + final picture = _regionDiff(withMask, src, 700, 300, 1800, 1000); + // ignore: avoid_print + print(' grid: protected $gridKept, unprotected $gridLost; picture $picture'); + expect(gridKept, lessThan(0.5), reason: 'the credits were not protected'); + expect(gridLost, greaterThan(20), reason: 'without the mask the grid should be lost — ' + 'if it is not, the mask is not what protects it'); + expect(picture, lessThan(3.0), reason: 'the rescaled picture is not the source\'s'); + + final pv = await WorkerHarness.runPreview(_job('e_pv', credits, ds).toJson(), + frame: 3, label: 'descale_pv'); + expect(pv.success, isTrue, reason: pv.errorTail); + final pvDiff = WorkerHarness.meanAbsDiff( + await WorkerHarness.imageToRgb24(pv.png!, label: 'descale_pv'), + await WorkerHarness.frameRgb24(protected.outputPath!, 3, label: 'descale_enc3')); + // ignore: avoid_print + print(' preview vs encode: $pvDiff'); + expect(pvDiff, lessThan(1.0)); + }, timeout: const Timeout(Duration(minutes: 15))); + }); +} diff --git a/app/test/integration_filter_parameters_test.dart b/app/test/integration_filter_parameters_test.dart index 2423487b..85f39776 100644 --- a/app/test/integration_filter_parameters_test.dart +++ b/app/test/integration_filter_parameters_test.dart @@ -20,6 +20,7 @@ import 'package:vapourbox/models/crop_resize_parameters.dart'; import 'package:vapourbox/models/deband_parameters.dart'; import 'package:vapourbox/models/deblock_parameters.dart'; import 'package:vapourbox/models/dehalo_parameters.dart'; +import 'package:vapourbox/models/descale_parameters.dart'; import 'package:vapourbox/models/descratch_parameters.dart'; import 'package:vapourbox/models/dynamic_parameters.dart'; import 'package:vapourbox/models/encoding_settings.dart'; @@ -173,6 +174,7 @@ VideoJob buildJob({ StabilizeParameters? stabilize, GeometryParameters? geometry, GrainParameters? grain, + DescaleParameters? descale, }) => VideoJob( id: const Uuid().v4(), inputPath: TestConfig.inputFile, @@ -194,6 +196,7 @@ VideoJob buildJob({ stabilize: stabilize ?? const StabilizeParameters(), geometry: geometry ?? const GeometryParameters(), grain: grain ?? const GrainParameters(), + descale: descale ?? const DescaleParameters(), ), encodingSettings: const EncodingSettings( codec: VideoCodec.h264, container: ContainerFormat.mkv, audioMode: AudioMode.none, @@ -1017,6 +1020,96 @@ void main() { print(' PASS'); }, timeout: const Timeout(Duration(minutes: 2))); + test('noise reduction: BM3D params reach the 16-bit per-plane block', + () async { + loadSchema('noise_reduction'); + final typed = const NoiseReductionParameters( + enabled: true, + method: NoiseReductionMethod.bm3d, + bm3dSigma: 4.5, + bm3dChromaSigma: 2.0, + bm3dRadius: 2, + bm3dFinal: false, + bm3dProfile: 'high', + ); + // Through the converter, as the panel would hand it to the worker. + final roundTripped = ParameterConverter.toNoiseReduction( + ParameterConverter.fromNoiseReduction(typed)); + final job = buildJob(testName: 'nr_bm3d', noiseReduction: roundTripped); + print(' Generating BM3D script...'); + final script = await generateScriptViaWorker(job); + expect(script, contains('core.bm3d.VBasic(')); + expect(script, contains('_bm3d_radius = 2')); + expect(script, contains('_bm3d_final = False')); + expect(script, contains('_bm3d_profile = "high"')); + expect(script, contains('_bm3d_sigmas = [4.50] + [2.00]')); + expect(script, isNot(contains('core.knlm.KNLMeansCL('))); + print(' PASS'); + }, timeout: const Timeout(Duration(minutes: 2))); + + test('noise reduction: KNLMeans runs on the probed device, or is refused', + () async { + loadSchema('noise_reduction'); + final probe = await Process.run( + WorkerHarness.workerPath, + ['--probe-opencl'], + environment: WorkerHarness.workerEnv, + workingDirectory: File(WorkerHarness.workerPath).parent.path, + ); + final caps = jsonDecode(const LineSplitter() + .convert(probe.stdout.toString()) + .lastWhere((l) => l.trim().startsWith('{'))) as Map; + print(' OpenCL probe: $caps'); + final job = buildJob( + testName: 'nr_knlm', + noiseReduction: const NoiseReductionParameters( + enabled: true, + method: NoiseReductionMethod.knlMeans, + knlmH: 2.5, + knlmD: 2, + ), + ); + final device = caps['knlmDevice']; + if (device is String) { + final script = await generateScriptViaWorker(job); + expect(script, contains('channels="Y", device_type="$device")')); + expect(script, contains('d=2, a=2, s=4, h=2.50')); + } else { + // No device here (headless CI): the job must be refused with the + // probe's own message, which is also what the panel shows. + expect(caps['knlmMessage'], isA()); + final result = await WorkerHarness.runJob(job.toJson(), label: 'nr_knlm'); + expect(result.success, isFalse); + expect('${result.error}\n${result.logs.join('\n')}', + contains(caps['knlmMessage'] as String)); + } + print(' PASS'); + }, timeout: const Timeout(Duration(minutes: 2))); + + test('descale: kernel pair, native size and protection reach the script', () async { + loadSchema('descale'); + const typed = DescaleParameters( + enabled: true, + nativeHeight: 540, + kernel: DescaleKernel.lanczos, + lanczosTaps: 4, + protectDetail: true, + detailThreshold: 6.5, + ); + // Through the converter, as the panel would hand it to the worker. + final roundTripped = + ParameterConverter.toDescale(ParameterConverter.fromDescale(typed)); + final job = buildJob(testName: 'descale', descale: roundTripped); + print(' Generating Descale script...'); + final script = await generateScriptViaWorker(job); + expect(script, contains('_ds_h = 540')); + expect(script, contains('return core.descale.Delanczos(c, w, h, taps=4, border_handling=0)')); + expect(script, contains('return core.resize.Lanczos(c, w, h, filter_param_a=4)')); + expect(script, contains('core.std.Binarize(_ds_error, 6.5 / 255.0)')); + expect(script, contains('_nnedi3(_ds_up, field=1, dh=True')); + print(' PASS'); + }, timeout: const Timeout(Duration(minutes: 2))); + test('sharpen: aWarpSharp2 params', () async { loadSchema('sharpen'); final typed = const SharpenParameters( diff --git a/app/test/integration_frame_mapping_test.dart b/app/test/integration_frame_mapping_test.dart index 14fec4cd..eea38d13 100644 --- a/app/test/integration_frame_mapping_test.dart +++ b/app/test/integration_frame_mapping_test.dart @@ -42,6 +42,7 @@ import 'package:vapourbox/models/processing_pipeline.dart'; import 'package:vapourbox/models/qtgmc_parameters.dart'; import 'package:vapourbox/models/video_job.dart'; +import 'support/gpu_expectations.dart'; import 'support/worker_harness.dart'; String get _outDir => '${WorkerHarness.outputDir}/frame_mapping'; @@ -379,9 +380,9 @@ void main() { } }); - // RIFE needs a Vulkan device, which the hosted runners lack, so this group - // runs only where `--probe-rife` says RIFE works (integration_rife_test.dart - // covers the refusal everywhere else). Its count must match FrameMap's + // RIFE needs a Vulkan device. CI provisions one wherever RIFE ships (see + // support/gpu_expectations.dart), so there this group must run; elsewhere it + // skips, and integration_gpu_plugins_test.dart covers the refusal. Its count must match FrameMap's // Retime exactly — floor(n * 2) here — and since every other output frame is // a real one passed through untouched, the preview of source frame S must be // output frame 2S, picture for picture. @@ -425,20 +426,15 @@ void main() { }); test('25 -> 50 emits exactly twice the frames', () async { - if (!available) { - markTestSkipped('RIFE is not available on this machine'); - return; - } + // Skips only where no Vulkan is expected; fails where CI provisions it. + if (!available && !await requireGpuPlugin('--probe-rife')) return; expect(await _countFrames(encoded), _interlacedFrames * 2); }); for (final sourceFrame in [12, 36]) { test('source frame $sourceFrame previews as output frame ${sourceFrame * 2}', () async { - if (!available) { - markTestSkipped('RIFE is not available on this machine'); - return; - } + if (!available && !await requireGpuPlugin('--probe-rife')) return; final pv = await WorkerHarness.runPreview(rifeJob().toJson(), frame: sourceFrame, label: 'rife_pv$sourceFrame'); expect(pv.success, isTrue, reason: pv.toString()); diff --git a/app/test/integration_gpu_plugins_test.dart b/app/test/integration_gpu_plugins_test.dart new file mode 100644 index 00000000..5e7653ab --- /dev/null +++ b/app/test/integration_gpu_plugins_test.dart @@ -0,0 +1,206 @@ +// RIFE and AI upscaling (Real-ESRGAN / Real-CUGAN), through the real worker. +// +// Both run a network through ncnn on Vulkan, and ncnn segfaults without a +// Vulkan device rather than raising. So what must hold on EVERY platform is +// that the worker knows when it can't run and refuses with a message the user +// can act on; and where it can run — which CI makes true on every platform that +// ships it, with a software Vulkan driver on the GPU-less runners (see +// support/gpu_expectations.dart) — that the script loads the plugin explicitly +// and a real frame comes out of the GPU at the right size. +// +// Runs in the per-push gate. Full encodes, frame counts and pixel checks are in +// integration_frame_mapping_test.dart (RIFE) and integration_ai_upscale_test.dart +// (heavy). +library; + +import 'dart:convert'; +import 'dart:io'; +import 'dart:typed_data'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:path/path.dart' as p; +import 'package:uuid/uuid.dart'; + +import 'package:vapourbox/models/crop_resize_parameters.dart'; +import 'package:vapourbox/models/encoding_settings.dart'; +import 'package:vapourbox/models/frame_rate_parameters.dart'; +import 'package:vapourbox/models/gpu_plugin_status.dart'; +import 'package:vapourbox/models/processing_pipeline.dart'; +import 'package:vapourbox/models/qtgmc_parameters.dart'; +import 'package:vapourbox/models/video_job.dart'; + +import 'support/gpu_expectations.dart'; +import 'support/worker_harness.dart'; + +VideoJob _job(String name, ProcessingPipeline pipeline) => VideoJob( + id: const Uuid().v4(), + inputPath: WorkerHarness.inputFile, + outputPath: p.join(WorkerHarness.outputDir, 'gpu', '$name.mkv'), + processingPipeline: pipeline, + encodingSettings: const EncodingSettings( + codec: VideoCodec.ffv1, + container: ContainerFormat.mkv, + audioMode: AudioMode.none, + ), + inputWidth: 720, + inputHeight: 576, + inputPixelFormat: 'yuv422p', + inputFrameRate: 25.0, + totalFrames: 75, + ); + +VideoJob _rifeJob() => _job( + 'rife', + const ProcessingPipeline( + deinterlace: QTGMCParameters(enabled: false), + frameRate: FrameRateParameters( + enabled: true, + target: FrameRateTarget.pal50, + method: FrameRateMethod.rife, + sourceFpsNum: 25, + sourceFpsDen: 1, + ), + ), + ); + +VideoJob _upscaleJob(UpscaleMethod method, {String noise = 'conservative'}) => _job( + 'upscale_${method.name}', + ProcessingPipeline( + deinterlace: const QTGMCParameters(enabled: false), + cropResize: CropResizeParameters( + enabled: true, + useIntegerUpscale: true, + upscaleMethod: method, + upscaleFactor: 2, + cuganNoise: noise, + ), + ), + ); + +/// Width and height from a PNG's IHDR chunk. +(int, int) _pngSize(Uint8List png) { + final b = ByteData.sublistView(png); + return (b.getUint32(16), b.getUint32(20)); +} + +class _Feature { + final GpuPlugin plugin; + final VideoJob Function() job; + final String dir; + final List scriptMust; + final (int, int) previewSize; + const _Feature(this.plugin, this.job, this.dir, this.scriptMust, this.previewSize); +} + +final _features = { + 'RIFE': _Feature(GpuPlugin.rife, _rifeJob, 'rife', [ + 'core.rife.RIFE(_rife_rgb, model_path=', + 'fps_num=50, fps_den=1, sc=True)', + 'core.misc.SCDetect(clip, threshold=0.1)', + ], (720, 576)), + 'Real-ESRGAN': _Feature(GpuPlugin.upscale, () => _upscaleJob(UpscaleMethod.realEsrgan), 'ncnn', [ + 'RealESRGANv2-animevideo-xsx2.onnx', + 'core.ncnn.Model(_ai_rgb, network_path=_ai_model, fp16=True', + ], (1440, 1152)), + 'Real-CUGAN': _Feature(GpuPlugin.upscale, () => _upscaleJob(UpscaleMethod.realCugan, noise: 'denoise3x'), 'ncnn', [ + 'up2x-latest-denoise3x.onnx', + 'core.ncnn.Model(_ai_rgb, network_path=_ai_model, fp16=True', + ], (1440, 1152)), +}; + +void main() { + setUpAll(() async { + await WorkerHarness.ensureReady(); + // ignore: avoid_print + print('GPU plugins on $platformId (expect Vulkan: $expectVulkan)'); + }); + + for (final plugin in GpuPlugin.values) { + group('${plugin.label} probe', () { + test('answers, consistently with where it ships', () async { + final probe = await probeGpuPlugin(plugin.probeFlag); + // ignore: avoid_print + print(' ${plugin.probeFlag}: $probe'); + expect(probe['available'], isA()); + if (!platformShipsGpuPlugins) { + expect(probe['reason'], 'unsupportedPlatform', + reason: '$platformId ships no ${plugin.label}, and must say so'); + } else if (expectVulkan) { + expect(probe['available'], isTrue, + reason: 'CI provisions Vulkan here: ${probe['message']}'); + } + if (probe['available'] != true) { + final status = GpuPluginStatus.fromJson(plugin, probe); + expect(status.message, isNotNull); + expect(status.message!.length, greaterThan(40)); + expect(status.message, contains(plugin.label)); + expect(status.reason, isNot('probeFailed'), + reason: 'the probe could not even run here, which hides the real ' + 'reason from the user: ${status.message}'); + } + }); + }); + } + + for (final MapEntry(key: name, value: f) in _features.entries) { + group(name, () { + test('a job where it cannot run is refused with the probe\'s message', () async { + const env = {'VAPOURBOX_DISABLE_VULKAN': '1'}; + final disabled = await probeGpuPlugin(f.plugin.probeFlag, extraEnv: env); + expect(disabled['available'], isFalse); + expect(Platform.environment['VAPOURBOX_DISABLE_VULKAN'], isNull, + reason: 'run this test without VAPOURBOX_DISABLE_VULKAN set'); + // Through the real encode entry point: it must fail before vspipe + // starts, naming the reason, rather than produce a script that would + // segfault. + final job = f.job(); + final configFile = File(p.join(Directory.systemTemp.path, 'vb_gpu_${job.id}.json')); + await configFile.writeAsString(jsonEncode(job.toJson())); + try { + final r = await Process.run( + WorkerHarness.workerPath, + ['--config', configFile.path], + environment: {...WorkerHarness.workerEnv, ...env}, + workingDirectory: File(WorkerHarness.workerPath).parent.path, + ); + final errors = const LineSplitter() + .convert(r.stdout.toString()) + .where((l) => l.contains('"type":"error"')) + .toList(); + expect(errors, isNotEmpty, reason: 'the job was not refused: ${r.stdout}'); + // Whatever the reason (disabled, or no build for this platform), + // the refusal is the probe's own wording. + final reason = GpuPluginStatus.fromJson(f.plugin, disabled).message!; + final messages = errors + .map((l) => (jsonDecode(l) as Map)['message'] as String) + .join('\n'); + expect(messages, contains(reason)); + expect(File(p.join(Directory.systemTemp.path, '${job.id}.vpy')).existsSync(), isFalse, + reason: 'a script was written for a job that cannot run'); + } finally { + await configFile.delete().catchError((_) => configFile); + } + }); + + test('where it runs, the script loads it explicitly and calls it', () async { + if (!await requireGpuPlugin(f.plugin.probeFlag)) return; + final script = await WorkerHarness.generateScript(f.job().toJson()); + expect(script, contains('core.std.LoadPlugin(')); + expect(script, contains('${p.separator}${f.dir}${p.separator}'.replaceAll(r'\', r'\\'))); + for (final must in f.scriptMust) { + expect(script, contains(must)); + } + if (platformId.startsWith('macos')) { + expect(script, contains('moltenvk'), reason: 'MoltenVK must be preloaded on macOS'); + } + }); + + test('where it runs, a real frame comes out of the GPU at the right size', () async { + if (!await requireGpuPlugin(f.plugin.probeFlag)) return; + final pv = await WorkerHarness.runPreview(f.job().toJson(), frame: 21, label: 'gpu_$name'); + expect(pv.success, isTrue, reason: pv.errorTail); + expect(_pngSize(pv.png!), f.previewSize); + }, timeout: const Timeout(Duration(minutes: 5))); + }); + } +} diff --git a/app/test/integration_high_bit_depth_filters_test.dart b/app/test/integration_high_bit_depth_filters_test.dart index 3b0c8ab0..4bf46817 100644 --- a/app/test/integration_high_bit_depth_filters_test.dart +++ b/app/test/integration_high_bit_depth_filters_test.dart @@ -49,6 +49,7 @@ import 'package:vapourbox/models/deband_parameters.dart'; import 'package:vapourbox/models/deblock_parameters.dart'; import 'package:vapourbox/models/dehalo_parameters.dart'; import 'package:vapourbox/models/deflicker_parameters.dart'; +import 'package:vapourbox/models/descale_parameters.dart'; import 'package:vapourbox/models/descratch_parameters.dart'; import 'package:vapourbox/models/edge_repair_parameters.dart'; import 'package:vapourbox/models/encoding_settings.dart'; @@ -278,6 +279,7 @@ ProcessingPipeline _only({ EdgeRepairParameters edgeRepair = const EdgeRepairParameters(), GhostRemovalParameters ghostRemoval = const GhostRemovalParameters(), FrameRateParameters frameRate = const FrameRateParameters(), + DescaleParameters descale = const DescaleParameters(), }) { return ProcessingPipeline( deinterlace: deinterlace, @@ -296,6 +298,7 @@ ProcessingPipeline _only({ edgeRepair: edgeRepair, ghostRemoval: ghostRemoval, frameRate: frameRate, + descale: descale, ); } @@ -417,6 +420,29 @@ List<_Pass> _passes() => [ ), ), + // Descale works in float on luma; a correct block gives the same picture + // at 10-bit as at 8-bit (the float conversion normalises every depth to + // 0-1). Protection is off: this fixture was never upscaled, so with it on + // the mask rightly keeps almost every pixel and the pass changes nothing. + _Pass( + 'descale', + _only(descale: const DescaleParameters( + enabled: true, nativeHeight: 216, protectDetail: false)), + ), + + // BM3D works in 16-bit and its sigma is in 8-bit units at every depth, + // so a correct block gives the same picture at 10-bit as at 8-bit. + _Pass( + 'noise_bm3d', + _only( + noiseReduction: const NoiseReductionParameters( + enabled: true, + method: NoiseReductionMethod.bm3d, + bm3dChromaSigma: 2.0, + ), + ), + ), + // Deflicker measures frame brightness against a windowed average, so its // correction is a ratio and should be depth-independent by construction. _Pass( diff --git a/app/test/integration_new_passes_test.dart b/app/test/integration_new_passes_test.dart index 94499ce1..6490f390 100644 --- a/app/test/integration_new_passes_test.dart +++ b/app/test/integration_new_passes_test.dart @@ -219,6 +219,49 @@ void main() { await _expectValidVideo(result); }, timeout: const Timeout(Duration(minutes: 6))); + test('noise reduction: BM3D runs end-to-end, chroma included', () async { + final job = _baseJob( + 'nr_bm3d', + pipeline: const ProcessingPipeline( + deinterlace: QTGMCParameters(enabled: false), + noiseReduction: NoiseReductionParameters( + enabled: true, + method: NoiseReductionMethod.bm3d, + bm3dChromaSigma: 2.0, + ), + ), + ); + final result = await WorkerHarness.runJob(job.toJson(), label: 'nr_bm3d'); + await _expectValidVideo(result); + }, timeout: const Timeout(Duration(minutes: 6))); + + test('noise reduction: KNLMeans runs end-to-end where OpenCL works', + () async { + final probe = await Process.run( + WorkerHarness.workerPath, + ['--probe-opencl'], + environment: WorkerHarness.workerEnv, + workingDirectory: File(WorkerHarness.workerPath).parent.path, + ); + final job = _baseJob( + 'nr_knlm', + pipeline: const ProcessingPipeline( + deinterlace: QTGMCParameters(enabled: false), + noiseReduction: NoiseReductionParameters( + enabled: true, + method: NoiseReductionMethod.knlMeans, + ), + ), + ); + final result = await WorkerHarness.runJob(job.toJson(), label: 'nr_knlm'); + if (probe.stdout.toString().contains('"knlmDevice":null')) { + // Covered by the push-gate test; here it is enough that it never crashes. + expect(result.success, isFalse); + return; + } + await _expectValidVideo(result); + }, timeout: const Timeout(Duration(minutes: 6))); + test('sharpen: aWarpSharp2 runs end-to-end', () async { final job = _baseJob( 'sharpen_awarpsharp2', diff --git a/app/test/integration_rife_test.dart b/app/test/integration_rife_test.dart deleted file mode 100644 index a670c88b..00000000 --- a/app/test/integration_rife_test.dart +++ /dev/null @@ -1,143 +0,0 @@ -// RIFE frame interpolation, through the real worker binary. -// -// Script-only (no encode), so it runs in the per-push gate on every platform — -// including the hosted runners, which have no GPU. That is the point of two of -// these: RIFE segfaults without a Vulkan device, so what must hold everywhere -// is that the worker *knows* when it can't run and refuses with a message the -// user can act on. The script-content test runs only where the probe says RIFE -// works; the encode itself is in integration_frame_mapping_test.dart (heavy). -library; - -import 'dart:convert'; -import 'dart:io'; - -import 'package:flutter_test/flutter_test.dart'; -import 'package:path/path.dart' as p; -import 'package:uuid/uuid.dart'; - -import 'package:vapourbox/models/encoding_settings.dart'; -import 'package:vapourbox/models/frame_rate_parameters.dart'; -import 'package:vapourbox/models/processing_pipeline.dart'; -import 'package:vapourbox/models/qtgmc_parameters.dart'; -import 'package:vapourbox/models/rife_status.dart'; -import 'package:vapourbox/models/video_job.dart'; - -import 'support/worker_harness.dart'; - -VideoJob _rifeJob() => VideoJob( - id: const Uuid().v4(), - inputPath: WorkerHarness.inputFile, - outputPath: p.join(WorkerHarness.outputDir, 'rife', 'rife_script.mkv'), - processingPipeline: const ProcessingPipeline( - deinterlace: QTGMCParameters(enabled: false), - frameRate: FrameRateParameters( - enabled: true, - target: FrameRateTarget.pal50, - method: FrameRateMethod.rife, - sourceFpsNum: 25, - sourceFpsDen: 1, - ), - ), - encodingSettings: const EncodingSettings( - codec: VideoCodec.ffv1, - container: ContainerFormat.mkv, - audioMode: AudioMode.none, - ), - inputWidth: 720, - inputHeight: 576, - inputPixelFormat: 'yuv422p', - inputFrameRate: 25.0, - totalFrames: 75, - ); - -Future> _probe({Map extraEnv = const {}}) async { - final r = await Process.run( - WorkerHarness.workerPath, - ['--probe-rife'], - environment: {...WorkerHarness.workerEnv, ...extraEnv}, - workingDirectory: File(WorkerHarness.workerPath).parent.path, - ); - expect(r.exitCode, 0, reason: '--probe-rife failed: ${r.stderr}'); - final line = const LineSplitter() - .convert(r.stdout.toString()) - .lastWhere((l) => l.trim().startsWith('{'), orElse: () => ''); - expect(line, isNotEmpty, reason: 'no JSON from --probe-rife: ${r.stdout}'); - return jsonDecode(line) as Map; -} - -void main() { - late Map probe; - - setUpAll(() async { - await WorkerHarness.ensureReady(); - probe = await _probe(); - // ignore: avoid_print - print('RIFE probe on this machine: $probe'); - }); - - test('--probe-rife answers, and a "no" always comes with a remedy', () { - expect(probe['available'], isA()); - final status = RifeStatus.fromJson(probe); - expect(status.available, probe['available']); - if (!status.available) { - expect(probe['reason'], isA()); - expect(status.message, isNotNull); - expect(status.message!.length, greaterThan(40)); - expect(status.reason, isNot('probeFailed'), - reason: 'the probe could not even run here, which hides the real ' - 'reason from the user: ${status.message}'); - } - }); - - test('a RIFE job where RIFE cannot run is refused with that message', () async { - final env = {'VAPOURBOX_DISABLE_VULKAN': '1'}; - final disabled = await _probe(extraEnv: env); - expect(disabled['available'], isFalse); - - // Through the real encode entry point: it must fail before vspipe starts, - // naming the reason, rather than producing a script that would segfault. - final saved = Platform.environment['VAPOURBOX_DISABLE_VULKAN']; - expect(saved, isNull, reason: 'run this test without VAPOURBOX_DISABLE_VULKAN set'); - final job = _rifeJob(); - final configFile = File(p.join(Directory.systemTemp.path, 'vb_rife_${job.id}.json')); - await configFile.writeAsString(jsonEncode(job.toJson())); - try { - final r = await Process.run( - WorkerHarness.workerPath, - ['--config', configFile.path], - environment: {...WorkerHarness.workerEnv, ...env}, - workingDirectory: File(WorkerHarness.workerPath).parent.path, - ); - final errors = const LineSplitter() - .convert(r.stdout.toString()) - .where((l) => l.contains('"type":"error"')) - .toList(); - expect(errors, isNotEmpty, reason: 'the job was not refused: ${r.stdout}'); - // Whatever the reason here (disabled, or no RIFE build for this - // platform at all), the refusal is the probe's own wording. - final reason = RifeStatus.fromJson(disabled).message!; - final messages = errors - .map((l) => (jsonDecode(l) as Map)['message'] as String) - .join('\n'); - expect(messages, contains(reason)); - expect(File(p.join(Directory.systemTemp.path, '${job.id}.vpy')).existsSync(), isFalse, - reason: 'a script was written for a job that cannot run'); - } finally { - await configFile.delete().catchError((_) => configFile); - } - }); - - test('where RIFE runs, the script loads it explicitly and calls it', () async { - if (probe['available'] != true) { - markTestSkipped('RIFE is not available here: ${probe['reason']}'); - return; - } - final script = await WorkerHarness.generateScript(_rifeJob().toJson()); - expect(script, contains('core.std.LoadPlugin(')); - expect(script, contains('${p.separator}rife${p.separator}'.replaceAll(r'\', r'\\'))); - expect(script, contains('core.rife.RIFE(_rife_rgb, model_path=')); - expect(script, contains('fps_num=50, fps_den=1, sc=True)')); - expect(script, contains('core.misc.SCDetect(clip, threshold=0.1)')); - expect(script, isNot(contains('core.mv.FlowFPS('))); - }); -} diff --git a/app/test/rife_notice_widget_test.dart b/app/test/knlm_notice_widget_test.dart similarity index 54% rename from app/test/rife_notice_widget_test.dart rename to app/test/knlm_notice_widget_test.dart index 817bcdfe..d57e3ef4 100644 --- a/app/test/rife_notice_widget_test.dart +++ b/app/test/knlm_notice_widget_test.dart @@ -1,7 +1,7 @@ -// The Frame Rate panel tells the user, the moment RIFE is chosen, why it can't -// run here and what to do — the worker's message, verbatim, with a way to -// check again once they have. RIFE has no fallback (a RIFE job without Vulkan -// is refused), so this notice is the user's only warning before a job fails. +// The Noise Reduction panel tells the user, the moment KNLMeans is chosen, that +// it can't run here and why: the worker's own message (the job would be +// refused with the same text). BM3D, and a machine where KNLMeans works, stay +// silent. import 'dart:convert'; import 'dart:io'; @@ -13,7 +13,6 @@ import 'package:shared_preferences/shared_preferences.dart'; import 'package:vapourbox/models/dynamic_parameters.dart'; import 'package:vapourbox/models/filter_registry.dart'; import 'package:vapourbox/models/processing_pipeline.dart'; -import 'package:vapourbox/models/rife_status.dart'; import 'package:vapourbox/services/advanced_mode_service.dart'; import 'package:vapourbox/viewmodels/main_viewmodel.dart'; import 'package:vapourbox/views/pass_settings/pass_settings_inline.dart'; @@ -29,7 +28,7 @@ void main() { SharedPreferences.setMockInitialValues({}); AdvancedModeService.instance.resetForTesting(); FilterRegistry.instance.registerFromJson( - jsonDecode(File('assets/filters/core/frame_rate.json').readAsStringSync()) + jsonDecode(File('assets/filters/core/noise_reduction.json').readAsStringSync()) as Map, ); vm = MainViewModel(runner: FakeJobRunner()); @@ -38,11 +37,11 @@ void main() { tearDown(() => vm.dispose()); void selectMethod(String method) => vm.updateDynamicParams( - 'frame_rate', + 'noise_reduction', DynamicParameters( - filterId: 'frame_rate', + filterId: 'noise_reduction', enabled: true, - values: {'method': method, 'target': 'pal50', 'blockSize': 16, 'overlap': 8}, + values: {'method': method}, ), ); @@ -55,7 +54,7 @@ void main() { child: const MaterialApp( home: Scaffold( body: SingleChildScrollView( - child: PassSettingsInline(passType: PassType.frameRate), + child: PassSettingsInline(passType: PassType.noiseReduction), ), ), ), @@ -69,44 +68,27 @@ void main() { Future settle(WidgetTester tester) => tester.pump(const Duration(seconds: 1)); - const message = 'RIFE needs a graphics card with Vulkan support, and none was found.'; - const unavailable = - RifeStatus(available: false, reason: 'noVulkan', message: message); + const message = 'KNLMeans runs on the graphics card through OpenCL, and none was found.'; - testWidgets('RIFE selected where it cannot run: the remedy and Check again', + testWidgets('KNLMeans selected where no OpenCL device works: the reason', (tester) async { - selectMethod('rife'); - vm.debugSetRifeStatus(unavailable); + selectMethod('knlmeans'); + vm.debugSetKnlmMethodMessage(message); await pumpPanel(tester); - expect(find.text(message), findsOneWidget); - expect(find.text('Check again'), findsOneWidget); await settle(tester); }); - testWidgets('while the probe runs, it says so', (tester) async { - selectMethod('rife'); - vm.debugSetRifeStatus(null, checking: true); + testWidgets('silent for BM3D, and for KNLMeans where it runs', (tester) async { + selectMethod('bm3d'); + vm.debugSetKnlmMethodMessage(message); await pumpPanel(tester); - - expect(find.textContaining('Checking whether RIFE can run'), findsOneWidget); expect(find.text(message), findsNothing); - await settle(tester); - }); - - testWidgets('silent when RIFE runs, or when another method is chosen', - (tester) async { - selectMethod('rife'); - vm.debugSetRifeStatus(const RifeStatus(available: true)); - await pumpPanel(tester); - expect(find.text('Check again'), findsNothing); - expect(find.textContaining('RIFE can run'), findsNothing); - selectMethod('flowFps'); - vm.debugSetRifeStatus(unavailable); + selectMethod('knlmeans'); + vm.debugSetKnlmMethodMessage(null); await pumpPanel(tester); expect(find.text(message), findsNothing); - expect(find.text('Check again'), findsNothing); await settle(tester); }); } diff --git a/app/test/pass_list_stages_test.dart b/app/test/pass_list_stages_test.dart index 88c8b932..29d00367 100644 --- a/app/test/pass_list_stages_test.dart +++ b/app/test/pass_list_stages_test.dart @@ -42,6 +42,7 @@ void main() { PassType.deflicker, PassType.descratch, PassType.spotless, + PassType.descale, PassType.noiseReduction, PassType.chromaDenoise, PassType.dehalo, diff --git a/app/test/rife_status_test.dart b/app/test/rife_status_test.dart deleted file mode 100644 index 5523ded4..00000000 --- a/app/test/rife_status_test.dart +++ /dev/null @@ -1,75 +0,0 @@ -// RIFE's status as the app reads it from `--probe-rife`, and the Frame Rate -// panel's decision about what to say. Pure unit tests; the probe itself is -// exercised against the real worker in integration_rife_test.dart. -import 'package:flutter_test/flutter_test.dart'; - -import 'package:vapourbox/models/dynamic_parameters.dart'; -import 'package:vapourbox/models/frame_rate_parameters.dart'; -import 'package:vapourbox/models/parameter_converter.dart'; -import 'package:vapourbox/models/rife_status.dart'; - -void main() { - group('RifeStatus.fromJson', () { - test('available', () { - final s = RifeStatus.fromJson({'available': true}); - expect(s.available, isTrue); - expect(s.message, isNull); - }); - - test('unavailable carries the worker message verbatim', () { - final s = RifeStatus.fromJson({ - 'available': false, - 'reason': 'missingLibrary', - 'detail': 'could not load vcomp140.dll', - 'message': 'RIFE needs the Microsoft Visual C++ Redistributable (x64)…', - }); - expect(s.available, isFalse); - expect(s.reason, 'missingLibrary'); - expect(s.message, startsWith('RIFE needs the Microsoft Visual C++')); - }); - - test('an unavailable answer without a message is never shown as blank', () { - final s = RifeStatus.fromJson({'available': false, 'reason': 'noVulkan'}); - expect(s.available, isFalse); - expect(s.message, isNotEmpty); - }); - }); - - group('rifeNoticeFor', () { - const ok = RifeStatus(available: true); - const no = RifeStatus(available: false, reason: 'noVulkan', message: 'no GPU'); - - test('says nothing unless RIFE is the chosen method', () { - for (final method in ['flowFps', 'duplicate', null]) { - for (final status in [ok, no, null]) { - expect(rifeNoticeFor(method: method, status: status, checking: false), - RifeNotice.none, reason: '$method / $status'); - } - } - }); - - test('RIFE selected: checking, then unavailable or silent', () { - expect(rifeNoticeFor(method: 'rife', status: null, checking: false), - RifeNotice.checking); - expect(rifeNoticeFor(method: 'rife', status: no, checking: true), - RifeNotice.checking); - expect(rifeNoticeFor(method: 'rife', status: no, checking: false), - RifeNotice.unavailable); - expect(rifeNoticeFor(method: 'rife', status: ok, checking: false), - RifeNotice.none); - }); - }); - - test('the converter round-trips every Frame Rate method', () { - for (final method in FrameRateMethod.values) { - final params = FrameRateParameters(enabled: true, method: method); - final dynamic = ParameterConverter.fromFrameRate(params); - expect(dynamic.values['method'], method.name); - expect(ParameterConverter.toFrameRate(dynamic).method, method); - } - // An unknown id from an old or hand-edited preset falls back to FlowFPS. - final unknown = const DynamicParameters( - filterId: 'frame_rate', enabled: true, values: {'method': 'bogus'}); - expect(ParameterConverter.toFrameRate(unknown).method, FrameRateMethod.flowFps); - }); -} diff --git a/app/test/support/gpu_expectations.dart b/app/test/support/gpu_expectations.dart new file mode 100644 index 00000000..49a0f6ec --- /dev/null +++ b/app/test/support/gpu_expectations.dart @@ -0,0 +1,77 @@ +// Where the ncnn-on-Vulkan features (RIFE, AI upscaling) must work, and the +// one rule every GPU test applies. +// +// The hosted CI runners have no GPU. The workflows give the Linux and Windows +// runners Mesa's software Vulkan driver (lavapipe) and set +// VAPOURBOX_EXPECT_VULKAN=1 wherever a Vulkan device is provisioned. Under that +// variable, on a platform that ships the feature, "not available" is a test +// failure — so a broken bundle, plugin or probe cannot hide behind a skip, which +// is how it would read on a runner that simply has no GPU. Without it (a local +// run, a machine with no driver) the GPU tests skip and say why. On a platform +// that does not ship the feature, the probe must say exactly that. +library; + +import 'dart:convert'; +import 'dart:ffi'; +import 'dart:io'; + +import 'package:flutter_test/flutter_test.dart'; + +import 'worker_harness.dart'; + +/// The platforms whose deps bundle ships RIFE and VSNCNN — the worker's +/// `gpu_plugin::platform_ships_*`, which attribution_test.dart pins to the +/// deps manifest. +const gpuPluginPlatforms = {'macos-arm64', 'windows-x64', 'linux-x64'}; + +/// This machine's deps platform id. +String get platformId { + final abi = Abi.current(); + if (abi == Abi.macosArm64) return 'macos-arm64'; + if (abi == Abi.macosX64) return 'macos-x64'; + if (abi == Abi.windowsX64) return 'windows-x64'; + if (abi == Abi.windowsArm64) return 'windows-arm64'; + if (abi == Abi.linuxX64) return 'linux-x64'; + if (abi == Abi.linuxArm64) return 'linux-arm64'; + return abi.toString(); +} + +/// Whether this run has a Vulkan device provisioned and so must exercise the +/// GPU features rather than skip them. +bool get expectVulkan => Platform.environment['VAPOURBOX_EXPECT_VULKAN'] == '1'; + +/// Whether this platform's bundle ships the GPU features at all. +bool get platformShipsGpuPlugins => gpuPluginPlatforms.contains(platformId); + +/// Run the worker's probe (`--probe-rife`, `--probe-upscale`) and return its JSON. +Future> probeGpuPlugin( + String flag, { + Map extraEnv = const {}, +}) async { + final r = await Process.run( + WorkerHarness.workerPath, + [flag], + environment: {...WorkerHarness.workerEnv, ...extraEnv}, + workingDirectory: File(WorkerHarness.workerPath).parent.path, + ); + expect(r.exitCode, 0, reason: '$flag failed: ${r.stderr}'); + final line = const LineSplitter() + .convert(r.stdout.toString()) + .lastWhere((l) => l.trim().startsWith('{'), orElse: () => ''); + expect(line, isNotEmpty, reason: 'no JSON from $flag: ${r.stdout}'); + return jsonDecode(line) as Map; +} + +/// The rule, for a test that needs the feature to run: true to go ahead; +/// false after marking the test skipped (feature not expected to work here); +/// a test failure if it is expected to work and the probe says otherwise. +Future requireGpuPlugin(String flag) async { + final probe = await probeGpuPlugin(flag); + if (probe['available'] == true) return true; + if (platformShipsGpuPlugins && expectVulkan) { + fail('$flag says unavailable on $platformId, where CI provisions Vulkan ' + '(VAPOURBOX_EXPECT_VULKAN=1): ${probe['reason']}: ${probe['message']}'); + } + markTestSkipped('$flag: not available on $platformId (${probe['reason']})'); + return false; +} diff --git a/app/test/vapoursynth_integration_test.dart b/app/test/vapoursynth_integration_test.dart index 01721326..2776e64a 100644 --- a/app/test/vapoursynth_integration_test.dart +++ b/app/test/vapoursynth_integration_test.dart @@ -129,7 +129,7 @@ required = ['std', 'resize', 'mv', 'znedi3', 'eedi3m', 'fmtc', 'descratch', 'vivtc', 'ttmpsm', 'tmedian', 'fft3dfilter', 'flux', 'bifrost', 'retinex', 'bwdif', 'fb', 'removedirt', 'dedot', 'lghost', 'zsmooth', - 'vszip'] + 'vszip', 'bm3d', 'descale'] # On ARM, `nnedi3` is load-bearing and `znedi3` is only the fallback: znedi3's # SIMD is x86-only, so the ARM bundles build it scalar and both the templates' diff --git a/docs/ENGINEERING_NOTES.md b/docs/ENGINEERING_NOTES.md index 49fb5828..0ce97625 100644 --- a/docs/ENGINEERING_NOTES.md +++ b/docs/ENGINEERING_NOTES.md @@ -2380,6 +2380,142 @@ architecture MIT (Megvii, ECCV2022-RIFE), plugin MIT (HolyWu), ncnn BSD-3 (Tencent), MoltenVK Apache-2.0 (Brenwill Workshop, from source headers; no NOTICE file upstream). +### BM3D and KNLMeans as Noise Reduction methods (2026-10-25) + +Probed against the bundle on an M1 Pro (macOS arm64), BM3D r12 built with the +script's own `build_plugin`: + +- **BM3D refuses 4:2:0 whenever chroma is processed** ("sub-sampled format is + not supported when chroma is processed"); `sigma=[s,0,0]` works. Running + each plane as a GRAY clip handles chroma at its own resolution, and costs + nothing measurable: 52.5 fps whole-clip vs 53.0 per-plane at 720x480. +- **`VAggregate` always outputs 16-bit** (from 8-bit, 10-bit and even float + input), and `VFinal` then fails with "input clip and clip "ref" must be of the + same format" unless the input is 16-bit too. Hence the 16-bit working format. +- **sigma does not depend on depth**: 8-bit vs 16-bit results differ by + 0.30/255 (rounding), and the heavy parity test measures 0.36/255 at 10-bit. +- Throughput at 720x480, luma only, vspipe (frames in parallel): + + | radius | Basic | Basic + Final | + |---|---|---| + | 0 | 52.5 fps | 23.0 fps | + | 1 | 34.0 fps | 14.3 fps | + | 2 | 24.3 fps | 9.7 fps | + + Defaults are radius 1 with Final. Timing frames with sequential `get_frame` + calls reads about 10x slower and is not a measure of anything. +- **KNLMeansCL's default `device_type="auto"` fails on this Mac** with + `oclUtilsGetPlaformDeviceIDs: CL_INVALID_VALUE`; `"gpu"` works (140 fps luma, + 58 fps luma + chroma at 720x480), `"cpu"`/`"accelerator"` find no platform. + The existing `knlm` probe (default device) correctly reports false here, and + QTGMC's `knlmeanscl` denoiser has always fallen back to DFTTest on Apple + Silicon as a result — it is havsfunc's call, so it was left alone. The new + method probes `gpu` then `auto` and passes the one that works. +- Depth parity for KNLMeans: 0.137/255 at 10-bit, 0.009 at 16-bit, against + 2.74/255 of denoising. + +Why BM3D was missing from four of five bundles: Linux's +`patch_meson_vs_include` only matched `incdir = include_directories(run_command(...))`, +and r12 writes the probe inside a list with `'include'`, so meson ran the host +python (no VapourSynth module) and failed at configure. macOS x64 never calls +`build_plugin` (its plugins come from Stefan-Olt's prebuilts, which has no +current BM3D), and the PyPI wheel for x86_64 is minos 15.0. Neither failure was +visible because no expected-plugins list named BM3D outside macOS arm64. + +### AI upscaling: Real-ESRGAN / Real-CUGAN through VSNCNN (2026-10-25) + +Runtime choice. Neither a maintained "RealESRGAN-ncnn" nor "RealCUGAN-ncnn" +VapourSynth plugin exists (Kiyamou's CUGAN plugin is Windows-only and last +released 2022); vs-mlrt's VSNCNN (GPL-3.0, active) runs any of its ONNX models +on ncnn/Vulkan, the same stack as RIFE. Only its Windows build is published, +so macOS arm64 and Linux x64 build it from the pinned tag: upstream's own +linux-ncnn.yml recipe on Linux; on macOS, Tencent's ncnn ships as static +frameworks with no CMake config, so the three sources are compiled directly +with `-undefined dynamic_lookup` (vkGetInstanceProcAddr from the preloaded +MoltenVK, exactly as for RIFE). It is an API 3 plugin; R78 still loads those. + +Models and licences. AnimeJaNai was ruled out: CC BY-NC-SA 4.0. Real-ESRGAN +(BSD-3, Xintao Wang) and Real-CUGAN (MIT, bilibili) are fine. Every model is +2.4-5.2 MB, so they are bundled rather than fetched on demand; vs-mlrt +publishes them only inside one ~810 MB solid 7z per release, so each deps +build downloads it once (pinned by hash), extracts the five files in one pass +(separate extractions each re-decompress the solid archive) and checks each. + +Measured on an M1 Pro, fp16 (fp32 is ~3x slower and within 0.06/255 +(ESRGAN) / 0.30/255 (CUGAN) of it): + +| | SD -> 2x | 1080p -> 2x | +|---|---|---| +| Real-ESRGAN animevideo x2 | ~10 fps | 1.6 fps (whole frame) | +| Real-CUGAN up2x | ~6 fps | out of GPU memory whole-frame; 0.67 fps in 512-px tiles | + +num_streams 2 adds ~10%; 4 adds nothing. Without Vulkan (MoltenVK swapped for +an unrelated dylib) VSNCNN raises "get_gpu_device failed" rather than +segfaulting. + +Pixel checks (heavy test), against a plain Spline36 upscale of the same frames: + +- On real footage the comparison cannot prove correctness. On the interlaced + VHS fixture without deinterlacing, both networks remove the combing (CUGAN + smooths it, ESRGAN sharpens it into artefacts), a 9-12 level difference that + survives downscaling back to source size because it IS source-resolution + detail. small_clip has duplicate frames (a neighbour can score 0.0), so it + cannot catch a wrong frame either. It is kept as a sanity bound only. +- A synthetic pattern can: mid-saturation flat patches (nothing clips, where + the networks roll off by a few levels and a range error would hide) and a + white bar. Real-CUGAN 2x reproduces flat colour within 1 level on every + patch, which — the conversions being shared — proves the matrix and range. + Real-ESRGAN tints/saturates flat colour: up to 3 levels at 2x and 11 at 4x + (green 160 -> 171), in fp32 too, with vs-mlrt's own `RealESRGAN()` wrapper + doing nothing more than this template; that is the model. CUGAN 4x drifts ~4. +- Geometry: the networks widen a hard edge (CUGAN 4x: bar 194.6 px wide vs + 192.05), so either edge alone looks shifted (CUGAN 4x left edge 1.65 px off) + while the bar's centre is within 0.36 px for every model. The test checks the + centre on both axes (0.75 px), and that Spline36 itself lands on factor x + source (within 0.25 px), so the reference is known-good. + +CI runs it all for real: Mesa lavapipe on the GPU-less Linux/Windows runners +(ncnn tags a CPU device as type 3 but does not skip it), the runner's Metal GPU +on macOS arm64, and `VAPOURBOX_EXPECT_VULKAN=1` so an unavailable feature is a +failure there, not a skip. + +### Descale and native-resolution detection (2026-10-25) + +Probed with descale r12 (wheel) on an M1 Pro. A float 720p image upscaled to +1080p with Catmull-Rom and descaled back: round-trip error 2.8e-8 at 720 vs +~2e-4 at every other height; at 720, Catmull-Rom 2.8e-8, Mitchell 1.9e-5, +bilinear 6.3e-5, Spline36 8.0e-5, Lanczos3 1.0e-4. A sequential 231-height +scan took 12 s; spliced into one clip and read with `frames()` the whole +1080p detection (4 kernels x 540 heights, then 12 kernels x 4 frames, plus +decoding) takes ~30 s. + +What went wrong on the way, so it is not repeated: + +- **One-sided neighbourhoods.** The first version picked H-1 (1079, 574) on + everything: the round-trip error falls steadily toward the source height, + and a window that only has neighbours below reads that slope as a dip. +- **A smooth fixture proves nothing.** testing on small_clip (240p) scaled to + "native" 720p: after 8-bit FFV1 storage every height's misfit (~0.05 levels) + is below rounding (~0.25), so no dip exists to find. testsrc2 at native 720p + works. +- **Credits swamp a mean.** A dense one-pixel grid over 4% of the frame + (misfit at every height) flattened the dip to ~0.9; capping each pixel's + error at 2 levels before averaging restores it (0.48 at 4% coverage, 0.35 at + 2%, 0.14 with none). +- **Scan with every kernel family.** Bilinear + Catmull-Rom alone missed + Lanczos- and spline-made masters entirely (dip 0.63 / 0.56); adding + Lanczos 3 and Spline36 to the scan finds all four at 720. ffmpeg's `spline` + is not Spline36; its master is detected as Spline64, the nearest family. +- **A drawgrid on a transparent layer draws nothing** (alpha is not written): + the first "credits" fixture had no credits, and protected/unprotected came + out identical. The heavy test now asserts the unprotected grid is destroyed. + +Results on the fixtures: Catmull-Rom master -> 720 bicubic 0/0.5 (dip 0.14); +bilinear -> 720 bilinear (0.14); Lanczos -> 720 lanczos 3 (0.16); with 2% +credits -> 720 (0.35); the VHS capture -> not found (0.91). Encode with +protection: credits exact (0.0 levels), picture within 0.27 of the source; +without it the credits move 62.6 levels. 8- vs 10-bit parity 0.32. + ### Testing a deps change before publishing A PR that changes `deps/` has a chicken-and-egg problem: `ci-test.yml` and diff --git a/licenses/NOTICES.txt b/licenses/NOTICES.txt index 7a59baf4..1e316c89 100644 --- a/licenses/NOTICES.txt +++ b/licenses/NOTICES.txt @@ -195,6 +195,10 @@ RIFE (VapourSynth-RIFE-ncnn-Vulkan) — MIT The binary statically includes: ncnn — BSD 3-Clause, Copyright (C) 2017 Tencent. All rights reserved. https://github.com/Tencent/ncnn + ncnn's LICENSE.txt also lists SIMD math headers it includes under the + zlib licence: neon_mathfun.h and sse_mathfun.h (Copyright (C) 2011 and + 2007 Julien Pommier) and avx_mathfun.h (Copyright (C) 2012 Giovanni + Garberoglio), each with Tencent modifications. glslang — the core under BSD 3-Clause (Copyright (C) 2015-2018 Google, Inc. and others), with parts under BSD 2-Clause, MIT and Apache-2.0 (Copyright 2020 The Khronos Group Inc), as its LICENSE.txt sets out @@ -215,8 +219,46 @@ MoltenVK — Apache-2.0 Copyright (c) 2015-2026 The Brenwill Workshop Ltd. https://github.com/KhronosGroup/MoltenVK Licence text: Apache-2.0.txt - Vulkan on Apple's Metal. macOS arm64 only, beside RIFE, which needs a Vulkan - implementation loaded before it. + Vulkan on Apple's Metal. macOS arm64 only, in vapoursynth/moltenvk/: RIFE + and VSNCNN both need a Vulkan implementation loaded before them. + +VSNCNN (vs-mlrt) — GPL-3.0 + No copyright notice is given upstream; published by AmusementClub + https://github.com/AmusementClub/vs-mlrt + Licence text: GPL-3.0.txt + Provides core.ncnn.Model, which runs Crop & Resize's AI upscaling + (Real-ESRGAN / Real-CUGAN). Shipped on macOS arm64, Windows x64 and Linux + x64 only, in vapoursynth/ncnn/: built from the v15.16 source on macOS and + Linux, vs-mlrt's own build on Windows. The binary statically includes: + ncnn — BSD 3-Clause, Copyright (C) 2017 Tencent. All rights reserved. + https://github.com/Tencent/ncnn + ncnn's LICENSE.txt also lists SIMD math headers it includes under the + zlib licence: neon_mathfun.h and sse_mathfun.h (Copyright (C) 2011 and + 2007 Julien Pommier) and avx_mathfun.h (Copyright (C) 2012 Giovanni + Garberoglio), each with Tencent modifications. + glslang — as in the RIFE entry above + Protocol Buffers — BSD 3-Clause, Copyright 2008 Google Inc. All rights + reserved. + https://github.com/protocolbuffers/protobuf + ONNX — Apache-2.0, Copyright (c) ONNX Project Contributors + https://github.com/onnx/onnx + Licence text: Apache-2.0.txt + LLVM OpenMP runtime (macOS only, from ncnn's macOS build) — Apache-2.0 + WITH LLVM-exception, Copyright (c) the LLVM Project contributors + https://github.com/llvm/llvm-project/tree/main/openmp + Licence text: Apache-2.0-LLVM-Exception.txt + +Real-ESRGAN anime video model — BSD 3-Clause + Copyright (c) 2021, Xintao Wang + https://github.com/xinntao/Real-ESRGAN + The trained network weights of realesr-animevideov3, at 2x and 4x. Shipped as + vs-mlrt's ONNX conversions (RealESRGANv2-animevideo-xsx2/xsx4.onnx). + +Real-CUGAN models — MIT + Copyright (c) 2022 bilibili + https://github.com/bilibili/ailab/tree/main/Real-CUGAN + The trained network weights of the 2x models (no-denoise, conservative, + denoise3x). Shipped as vs-mlrt's ONNX conversions (up2x-latest-*.onnx). DFTTest — GPL-3.0 Copyright (C) 2007-2010 Kevin Stone (tritical); VapourSynth port by HolyWu @@ -279,7 +321,13 @@ Retinex — GPL-3.0 BM3D — MIT Copyright (c) 2015-2016 mawen1250 https://github.com/HomeOfVapourSynthEvolution/VapourSynth-BM3D - Shipped on macOS arm64 only. + +descale — MIT + Copyright © 2017-2022 Frechdachs (the LICENSE file says © 2021 Frechdachs); + maintained by Jaded Encoding Thaumaturgy + https://github.com/Jaded-Encoding-Thaumaturgy/vapoursynth-descale + Provides core.descale.De*, which inverts a resampling kernel: the Descale + pass. Taken unmodified from the project's PyPI wheels. KNLMeansCL — GPL-3.0 Copyright (C) 2015-2020 Edoardo Brunetti @@ -462,8 +510,9 @@ Appendix — full text of the short permissive licences MIT License ----------- (Applies to zsmooth, vszip and the zigimg library it includes, CAS, -DCTFilter, BM3D, the RIFE plugin, rife-ncnn-vulkan, the RIFE model and -ECCV2022-RIFE, whisper.cpp, pip and the Clense filter in vs-removegrain, each +DCTFilter, BM3D, descale, the RIFE plugin, rife-ncnn-vulkan, the RIFE model and +ECCV2022-RIFE, the Real-CUGAN models, whisper.cpp, pip and the Clense filter +in vs-removegrain, each under the copyright notice given in its entry above.) Permission is hereby granted, free of charge, to any person obtaining a copy @@ -523,8 +572,8 @@ DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE, Version 2 BSD 3-Clause License -------------------- -(Applies to Flutter, Zstandard, ncnn and the core of glslang, under the -copyright notices given above.) +(Applies to Flutter, Zstandard, ncnn, the core of glslang, Protocol Buffers +and the Real-ESRGAN model, under the copyright notices given above.) Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -580,6 +629,8 @@ DEALINGS IN THE SOFTWARE. zlib License ------------ +(Applies to zlib and to the SIMD math headers ncnn includes, under the +copyright notices given above.) This software is provided 'as-is', without any express or implied warranty. In no event will the authors be held liable for any damages arising from the use diff --git a/worker/src/dependency_locator.rs b/worker/src/dependency_locator.rs index b5e5f840..2b958f09 100644 --- a/worker/src/dependency_locator.rs +++ b/worker/src/dependency_locator.rs @@ -7,6 +7,15 @@ use std::sync::OnceLock; use anyhow::{bail, Context, Result}; +use crate::gpu_plugin::{self, GpuPlugin, GpuPluginFiles, GpuPluginReason, GpuPluginStatus}; + +/// Why the KNLMeans noise-reduction method can't run, as the app shows it. +pub const KNLM_NO_DEVICE_MESSAGE: &str = "KNLMeans runs on the graphics card through OpenCL, \ +and no OpenCL device that works with it was found on this computer. Updating the graphics \ +driver may fix this. BM3D does the same job on the CPU and needs nothing extra."; +pub const KNLM_DISABLED_MESSAGE: &str = "KNLMeans is turned off on this computer \ +(VAPOURBOX_DISABLE_OPENCL is set). BM3D does the same job on the CPU."; + /// Platform-specific dependency locator. pub struct DependencyLocator { base_path: PathBuf, @@ -295,7 +304,7 @@ impl DependencyLocator { } } - let result = self.probe_knlm(); + let result = self.probe_knlm(None); let _ = std::fs::write( &cache_file, serde_json::json!({ "boot": boot, "knlm": result }).to_string(), @@ -303,61 +312,125 @@ impl DependencyLocator { result } - /// RIFE's plugin, model and (macOS) MoltenVK, in this deps bundle. - pub fn rife_files(&self) -> crate::rife::RifeFiles { - crate::rife::RifeFiles::in_vapoursynth_dir( + /// The OpenCL `device_type` VapourBox's own KNLMeans call can use here, if + /// any: `gpu`, else the plugin's default `auto` (which also finds CPU + /// OpenCL runtimes). This is not [`knlm_available`]: that one answers for + /// havsfunc's call, which always passes the default — and the default + /// fails on Apple Silicon (`CL_INVALID_VALUE`) where `gpu` works. Cached by + /// boot time, in its own file so neither probe can read the other's answer. + pub fn knlm_device(&self) -> Option { + static CACHE: OnceLock> = OnceLock::new(); + CACHE.get_or_init(|| self.compute_knlm_device()).clone() + } + + fn compute_knlm_device(&self) -> Option { + if env::var("VAPOURBOX_DISABLE_OPENCL") + .map(|v| v == "1" || v.eq_ignore_ascii_case("true")) + .unwrap_or(false) + { + return None; + } + + let boot = Self::boot_token(); + let cache_file = self.platform_dir().join("knlm-device-probe.json"); + if !boot.is_empty() { + if let Ok(txt) = std::fs::read_to_string(&cache_file) { + if let Ok(json) = serde_json::from_str::(&txt) { + if json.get("boot").and_then(|v| v.as_str()) == Some(boot.as_str()) { + if let Some(device) = json.get("device") { + return device.as_str().map(str::to_string); + } + } + } + } + } + + let result = ["gpu", "auto"] + .into_iter() + .find(|d| self.probe_knlm(Some(d))) + .map(str::to_string); + let _ = std::fs::write( + &cache_file, + serde_json::json!({ "boot": boot, "device": result }).to_string(), + ); + result + } + + /// The device for a KNLMeans noise-reduction job, or the message the app + /// shows and the worker refuses that job with. + pub fn knlm_runtime(&self) -> Result { + self.knlm_device().ok_or_else(|| { + if env::var("VAPOURBOX_DISABLE_OPENCL").is_ok_and(|v| v == "1" || v.eq_ignore_ascii_case("true")) { + KNLM_DISABLED_MESSAGE.to_string() + } else { + KNLM_NO_DEVICE_MESSAGE.to_string() + } + }) + } + + /// A GPU plugin's files (plugin, models, macOS MoltenVK) in this bundle. + pub fn gpu_plugin_files(&self, plugin: GpuPlugin) -> GpuPluginFiles { + GpuPluginFiles::in_vapoursynth_dir( + plugin, &self.platform_dir().join("vapoursynth"), self.platform_suffix(), ) } - /// Whether RIFE runs here, and if not, why (see `rife.rs`). + /// Whether a GPU plugin runs here, and if not, why (see `gpu_plugin.rs`). /// /// Unlike the OpenCL probes, only a *positive* result is cached for the /// boot. A failure is what the user is told to fix — install a driver, the /// VC++ runtime — and then "restart VapourBox"; a failure cached until the /// next reboot would keep telling them it is broken after they fixed it. - /// Re-probing costs a second, and only on a machine where RIFE fails. + /// Re-probing costs a second, and only on a machine where it fails. /// `VAPOURBOX_DISABLE_VULKAN=1` forces a negative result (CI, and to try /// the unavailable path on a machine that has Vulkan). - pub fn rife_status(&self) -> crate::rife::RifeStatus { - static CACHE: OnceLock = OnceLock::new(); - CACHE.get_or_init(|| self.compute_rife_status()).clone() - } - - /// RIFE's files if it runs here, else the message saying why not — what - /// `ScriptGenerator::with_rife` takes. Runs the probe, so call it only for - /// a job that uses RIFE. - pub fn rife_runtime(&self) -> std::result::Result { - let status = self.rife_status(); - match status.message(self.platform_suffix()) { - None => Ok(self.rife_files()), + pub fn gpu_plugin_status(&self, plugin: GpuPlugin) -> GpuPluginStatus { + static RIFE: OnceLock = OnceLock::new(); + static UPSCALE: OnceLock = OnceLock::new(); + let cache = match plugin { + GpuPlugin::Rife => &RIFE, + GpuPlugin::Upscale => &UPSCALE, + }; + cache.get_or_init(|| self.compute_gpu_plugin_status(plugin)).clone() + } + + /// The plugin's files if it runs here, else the message saying why not — + /// what `ScriptGenerator::with_rife` / `with_upscale` take. Runs the probe, + /// so call it only for a job that uses the plugin. + pub fn gpu_plugin_runtime( + &self, + plugin: GpuPlugin, + ) -> std::result::Result { + match self.gpu_plugin_status(plugin).message(plugin, self.platform_suffix()) { + None => Ok(self.gpu_plugin_files(plugin)), Some(message) => Err(message), } } - fn compute_rife_status(&self) -> crate::rife::RifeStatus { - use crate::rife::{RifeReason, RifeStatus}; - let files = self.rife_files(); - if let Some(status) = crate::rife::precheck(self.platform_suffix(), &files) { + fn compute_gpu_plugin_status(&self, plugin: GpuPlugin) -> GpuPluginStatus { + let files = self.gpu_plugin_files(plugin); + if let Some(status) = gpu_plugin::precheck(plugin, self.platform_suffix(), &files) { return status; } if env::var("VAPOURBOX_DISABLE_VULKAN") .map(|v| v == "1" || v.eq_ignore_ascii_case("true")) .unwrap_or(false) { - return RifeStatus::unavailable(RifeReason::Disabled, None); + return GpuPluginStatus::unavailable(GpuPluginReason::Disabled, None); } let boot = Self::boot_token(); - let cache_file = self.platform_dir().join("rife-probe.json"); + let key = plugin.namespace(); + let cache_file = self.platform_dir().join(format!("{key}-probe.json")); if !boot.is_empty() { if let Ok(txt) = std::fs::read_to_string(&cache_file) { if let Ok(json) = serde_json::from_str::(&txt) { if json.get("boot").and_then(|v| v.as_str()) == Some(boot.as_str()) - && json.get("rife").and_then(|v| v.as_bool()) == Some(true) + && json.get("available").and_then(|v| v.as_bool()) == Some(true) { - return RifeStatus::available(); + return GpuPluginStatus::available(); } } } @@ -365,14 +438,19 @@ impl DependencyLocator { let vspipe = match self.vspipe_path() { Ok(p) => p, - Err(e) => return RifeStatus::unavailable(RifeReason::ProbeFailed, e.to_string()), + Err(e) => return GpuPluginStatus::unavailable(GpuPluginReason::ProbeFailed, e.to_string()), }; - let status = - crate::rife::run_probe(&vspipe, &self.build_environment(), &files, self.platform_suffix()); + let status = gpu_plugin::run_probe( + plugin, + &vspipe, + &self.build_environment(), + &files, + self.platform_suffix(), + ); if status.available { let _ = std::fs::write( &cache_file, - serde_json::json!({ "boot": boot, "rife": true }).to_string(), + serde_json::json!({ "boot": boot, "available": true }).to_string(), ); } else { let _ = std::fs::remove_file(&cache_file); @@ -382,7 +460,7 @@ impl DependencyLocator { /// Run KNLMeansCL through vspipe on a tiny clip; success ⇒ knlmeanscl usable /// (plugin present AND a usable OpenCL device). - fn probe_knlm(&self) -> bool { + fn probe_knlm(&self, device_type: Option<&str>) -> bool { let vspipe = match self.vspipe_path() { Ok(p) => p, Err(_) => return false, @@ -390,8 +468,10 @@ impl DependencyLocator { let script = "import vapoursynth as vs\n\ core = vs.core\n\ clip = core.std.BlankClip(width=256, height=256, format=vs.YUV420P8, length=1)\n\ - clip = core.knlm.KNLMeansCL(clip, d=0, h=1.0)\n\ + clip = core.knlm.KNLMeansCL(clip, d=0, h=1.0{device})\n\ clip.set_output()\n"; + let device = device_type.map(|d| format!(", device_type='{d}'")).unwrap_or_default(); + let script = script.replace("{device}", &device); let tmp = env::temp_dir().join(format!("vapourbox_knlm_probe_{}.vpy", std::process::id())); if std::fs::write(&tmp, script).is_err() { return false; diff --git a/worker/src/gpu_plugin.rs b/worker/src/gpu_plugin.rs new file mode 100644 index 00000000..d7a90e45 --- /dev/null +++ b/worker/src/gpu_plugin.rs @@ -0,0 +1,765 @@ +//! The ncnn-on-Vulkan plugins: where their files live, and whether they can run +//! here. +//! +//! Two features run a neural network through ncnn on the GPU: RIFE frame +//! interpolation (`rife.RIFE`, the Frame Rate pass) and AI upscaling +//! (vs-mlrt's `ncnn.Model` with Real-ESRGAN / Real-CUGAN, the Crop & Resize +//! pass). Both need more than the deps bundle — a GPU with a working Vulkan +//! driver — and without a Vulkan implementation ncnn does not raise: it +//! segfaults inside `ncnn::create_gpu_instance`. So a job cannot be allowed to +//! find out by trying. Instead this module runs the plugin once in a throwaway +//! vspipe, records how far it got, and turns the outcome into a reason with a +//! remedy the user can act on. The app shows that message the moment the +//! feature is selected (`--probe-rife`, `--probe-upscale`), and the worker +//! refuses such a job with the same message rather than emitting a script that +//! would crash. +//! +//! Each plugin lives in its own directory under `vapoursynth/`, outside the +//! autoload directory, and the script loads it explicitly — so nothing about it +//! (a missing runtime DLL, a driver that faults on load, AVX in a static +//! initializer) can reach a job that did not ask for it. On macOS both resolve +//! Vulkan from MoltenVK, which is preloaded from `vapoursynth/moltenvk/`. + +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::io::Read; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::time::{Duration, Instant}; + +/// RIFE's bundled model: 4.26, ncnn conversion, ensemble off. +pub const RIFE_MODEL_DIR_NAME: &str = "rife-v4.26_ensembleFalse"; + +/// The upscaling models, as paths under `vapoursynth/ncnn/models/` — vs-mlrt's +/// ONNX conversions, kept under the names vs-mlrt publishes them by. +pub const UPSCALE_MODELS: &[&str] = &[ + "RealESRGANv2/RealESRGANv2-animevideo-xsx2.onnx", + "RealESRGANv2/RealESRGANv2-animevideo-xsx4.onnx", + "cugan/up2x-latest-no-denoise.onnx", + "cugan/up2x-latest-conservative.onnx", + "cugan/up2x-latest-denoise3x.onnx", +]; + +/// How long the probe may take. GPU initialisation and the first shader +/// compile take a second or two; a driver that hangs must not hang the app. +const PROBE_TIMEOUT: Duration = Duration::from_secs(60); + +/// One of the ncnn-on-Vulkan features. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GpuPlugin { + /// RIFE frame interpolation. + Rife, + /// Real-ESRGAN / Real-CUGAN upscaling through vs-mlrt's VSNCNN. + Upscale, +} + +impl GpuPlugin { + /// The name the user knows it by, at the start of every message. + pub fn label(self) -> &'static str { + match self { + GpuPlugin::Rife => "RIFE", + GpuPlugin::Upscale => "AI upscaling", + } + } + + /// Its directory under `vapoursynth/`. + fn dir_name(self) -> &'static str { + match self { + GpuPlugin::Rife => "rife", + GpuPlugin::Upscale => "ncnn", + } + } + + /// The VapourSynth namespace the plugin registers. + pub fn namespace(self) -> &'static str { + match self { + GpuPlugin::Rife => "rife", + GpuPlugin::Upscale => "ncnn", + } + } + + /// What to use instead, for the reasons the user cannot fix. + fn alternative(self) -> &'static str { + match self { + GpuPlugin::Rife => "Use Motion interpolation instead.", + GpuPlugin::Upscale => "Use NNEDI3 upscaling instead.", + } + } + + /// Marker the probe script prints to stderr as it passes each stage, so a + /// process that died without a message still says where it died. + fn stage_marker(self) -> &'static str { + match self { + GpuPlugin::Rife => "RIFE_PROBE:", + GpuPlugin::Upscale => "UPSCALE_PROBE:", + } + } +} + +/// Whether the deps bundle for this platform ships the plugin at all. +/// +/// RIFE: macOS x64 is excluded because the only prebuilt dylib targets macOS 13 +/// (the bundle's floor is 12) and runs AVX in a static initializer; linux-arm64 +/// because upstream publishes no build. Both need a source build first. +pub fn platform_ships_rife(platform_suffix: &str) -> bool { + matches!(platform_suffix, "macos-arm64" | "windows-x64" | "linux-x64") +} + +/// AI upscaling: VSNCNN is built from source on macOS arm64 and Linux x64 (only +/// Windows has a published build). Neither Intel Macs nor linux-arm64 have the +/// MoltenVK copy / ncnn release this needs yet, as for RIFE. +pub fn platform_ships_upscale(platform_suffix: &str) -> bool { + matches!(platform_suffix, "macos-arm64" | "windows-x64" | "linux-x64") +} + +fn platform_ships(plugin: GpuPlugin, platform_suffix: &str) -> bool { + match plugin { + GpuPlugin::Rife => platform_ships_rife(platform_suffix), + GpuPlugin::Upscale => platform_ships_upscale(platform_suffix), + } +} + +/// The files a plugin needs, as laid out under `deps//vapoursynth/`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct GpuPluginFiles { + pub plugin: PathBuf, + /// RIFE's model directory, or the upscaling models' root. + pub model_dir: PathBuf, + /// A library to load into the process before the plugin: MoltenVK on + /// macOS, where ncnn resolves Vulkan from whatever is already loaded. + pub preload: Option, + /// The model files inside `model_dir` that must exist. + pub models: Vec, +} + +impl GpuPluginFiles { + pub fn in_vapoursynth_dir(plugin: GpuPlugin, vs_dir: &Path, platform_suffix: &str) -> Self { + let dir = vs_dir.join(plugin.dir_name()); + let os = platform_suffix.split('-').next().unwrap_or(""); + // The names upstream ships them under (RIFE's Windows DLL keeps "lib"). + let file = match (plugin, os) { + (GpuPlugin::Rife, "macos") => "librife.dylib", + (GpuPlugin::Rife, "windows") => "librife.dll", + (GpuPlugin::Rife, _) => "librife.so", + (GpuPlugin::Upscale, "macos") => "libvsncnn.dylib", + (GpuPlugin::Upscale, "windows") => "vsncnn.dll", + (GpuPlugin::Upscale, _) => "libvsncnn.so", + }; + let preload = platform_suffix + .starts_with("macos") + .then(|| vs_dir.join("moltenvk").join("libMoltenVK.dylib")); + let (model_dir, models) = match plugin { + GpuPlugin::Rife => { + let d = dir.join("models").join(RIFE_MODEL_DIR_NAME); + let m = vec![d.join("flownet.param"), d.join("flownet.bin")]; + (d, m) + } + GpuPlugin::Upscale => { + let d = dir.join("models"); + let m = UPSCALE_MODELS.iter().map(|m| d.join(m)).collect(); + (d, m) + } + }; + Self { plugin: dir.join(file), model_dir, preload, models } + } + + /// Every file that must exist, in the order a user would want to hear + /// about them. + pub fn required(&self) -> Vec { + let mut files = vec![self.plugin.clone()]; + files.extend(self.preload.iter().cloned()); + files.extend(self.models.iter().cloned()); + files + } + + pub fn missing(&self) -> Vec { + self.required().into_iter().filter(|p| !p.is_file()).collect() + } +} + +/// Why a plugin cannot run here. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum GpuPluginReason { + /// No build of it ships for this platform. + UnsupportedPlatform, + /// The installed deps bundle predates it, or is missing its files. + NotInstalled, + /// No usable Vulkan device or driver. + NoVulkan, + /// A system library the plugin links against is not installed. + MissingLibrary, + /// The plugin would not load, for a reason the probe could not narrow. + LoadFailed, + /// The probe process crashed after the GPU was initialised. + Crashed, + /// The plugin used an instruction this CPU does not have (SIGILL, or + /// Windows' 0xC000001D) — an x86 machine older than its build target. + UnsupportedCpu, + /// `VAPOURBOX_DISABLE_VULKAN` is set. + Disabled, + /// The probe itself could not be run or did not finish. + ProbeFailed, +} + +/// Whether a plugin runs here, and if not, why. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct GpuPluginStatus { + pub available: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reason: Option, + /// The specific thing that failed (a missing file, a library name, the + /// error text), shown under the remedy. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub detail: Option, +} + +impl GpuPluginStatus { + pub fn available() -> Self { + Self { available: true, reason: None, detail: None } + } + + pub fn unavailable(reason: GpuPluginReason, detail: impl Into>) -> Self { + Self { available: false, reason: Some(reason), detail: detail.into() } + } + + /// What to tell the user: what is wrong and what to do about it. `None` + /// when the plugin is available. The app shows this verbatim and the worker + /// fails such a job with it, so there is one wording, written here. + pub fn message(&self, plugin: GpuPlugin, platform_suffix: &str) -> Option { + use GpuPluginReason as R; + let reason = self.reason?; + let name = plugin.label(); + let os = platform_suffix.split('-').next().unwrap_or(""); + let text = match reason { + R::UnsupportedPlatform => { + let which = match platform_suffix { + "macos-x64" => "Intel Macs", + "linux-arm64" => "Linux on ARM", + "windows-arm64" => "Windows on ARM", + _ => "this platform", + }; + format!( + "{name} isn't available for {which} yet: no build of it ships with \ + VapourBox for this platform. {}", + plugin.alternative() + ) + } + R::NotInstalled => format!( + "{name}'s files aren't in the installed dependency bundle, which is \ + probably older than this version of VapourBox. Restart VapourBox to let \ + it download the current dependencies." + ), + R::NoVulkan => match os { + "windows" => format!( + "{name} needs a graphics card with Vulkan support, and none was \ + found. Install or update the driver from your GPU's maker (NVIDIA, \ + AMD or Intel) — Vulkan support comes with it — then restart \ + VapourBox. Remote desktop sessions and virtual machines usually have \ + no Vulkan device." + ), + "linux" => format!( + "{name} needs a graphics card with a Vulkan driver, and none was \ + found. Install your distribution's Vulkan driver and loader (for \ + example mesa-vulkan-drivers and libvulkan1 on Debian/Ubuntu, or \ + mesa-vulkan-drivers and vulkan-loader on Fedora; NVIDIA's own driver \ + includes Vulkan), then restart VapourBox." + ), + _ => format!( + "{name} could not open the GPU through Metal. This happens in virtual \ + machines and some remote sessions; on a normal login it should work, \ + so try restarting VapourBox." + ), + }, + R::MissingLibrary => match os { + "windows" => format!( + "{name} needs the Microsoft Visual C++ Redistributable (x64), which \ + isn't installed. Install the latest version from Microsoft (search \ + for \"Visual C++ Redistributable latest supported downloads\"), then \ + restart VapourBox." + ), + _ => format!( + "{name} needs a system library that isn't installed. Install it with \ + your package manager (libgomp is in the libgomp1 or libgomp package), \ + then restart VapourBox." + ), + }, + R::LoadFailed => format!( + "{name}'s plugin failed to load on this computer. Restart VapourBox to \ + re-check its dependencies; if this persists, please report it with the \ + details below." + ), + R::Crashed => format!( + "{name} crashed while testing the GPU. Updating your graphics driver \ + usually fixes this; restart VapourBox afterwards." + ), + R::UnsupportedCpu => format!( + "{name}'s plugin needs a newer processor than this computer has (it uses \ + instructions this CPU lacks). {}", + plugin.alternative() + ), + R::Disabled => format!( + "{name} is turned off by the VAPOURBOX_DISABLE_VULKAN environment variable." + ), + R::ProbeFailed => format!( + "VapourBox couldn't check whether {name} runs on this computer. Restart \ + VapourBox to try again." + ), + }; + Some(match &self.detail { + Some(d) if !d.is_empty() => format!("{text}\n\nDetails: {d}"), + _ => text, + }) + } + + /// The probe result as the JSON the `--probe-*` flags print, message included. + pub fn to_json(&self, plugin: GpuPlugin, platform_suffix: &str) -> serde_json::Value { + let mut v = serde_json::to_value(self).unwrap_or_default(); + if let (Some(msg), Some(obj)) = (self.message(plugin, platform_suffix), v.as_object_mut()) { + obj.insert("message".into(), serde_json::Value::String(msg)); + } + v + } +} + +/// Everything the platform and file checks can decide without running +/// anything. `None` means "it should work — run the probe". +pub fn precheck( + plugin: GpuPlugin, + platform_suffix: &str, + files: &GpuPluginFiles, +) -> Option { + if !platform_ships(plugin, platform_suffix) { + return Some(GpuPluginStatus::unavailable(GpuPluginReason::UnsupportedPlatform, None)); + } + let missing = files.missing(); + if !missing.is_empty() { + let list = missing.iter().map(|p| p.display().to_string()).collect::>(); + return Some(GpuPluginStatus::unavailable( + GpuPluginReason::NotInstalled, + format!("missing {}", list.join(", ")), + )); + } + None +} + +/// A Python string literal for a path, for substitution into a script. +pub fn py_str(path: &Path) -> String { + let s = path.to_string_lossy(); + let mut out = String::with_capacity(s.len() + 2); + out.push('\''); + for c in s.chars() { + match c { + '\\' => out.push_str("\\\\"), + '\'' => out.push_str("\\'"), + '\n' => out.push_str("\\n"), + '\r' => out.push_str("\\r"), + c => out.push(c), + } + } + out.push('\''); + out +} + +/// The probe: load the plugin, create the GPU instance, and render one frame — +/// the three places it has been seen to fail. +pub fn probe_script(plugin: GpuPlugin, files: &GpuPluginFiles, platform_suffix: &str) -> String { + // Shared libraries whose absence makes the plugin fail to load with only a + // generic "module not found". Checked first so the user is told which one. + // VSNCNN's Windows build links its runtime statically. + let runtime_libs: &[&str] = match plugin { + GpuPlugin::Rife if platform_suffix.starts_with("windows") => &[ + "vcomp140.dll", + "msvcp140.dll", + "msvcp140_atomic_wait.dll", + "vcruntime140.dll", + "vcruntime140_1.dll", + ], + _ if platform_suffix.starts_with("linux") => &["libgomp.so.1"], + _ => &[], + }; + let libs = runtime_libs.iter().map(|l| format!("'{l}'")).collect::>().join(", "); + let preload = files.preload.as_deref().map(py_str).unwrap_or_else(|| "None".into()); + // The call that makes ncnn create its GPU instance and run a network. + let call = match plugin { + GpuPlugin::Rife => format!( + "clip = core.rife.RIFE(clip, model_path={}, fps_num=50, fps_den=1)", + py_str(&files.model_dir) + ), + // The smallest bundled network. + GpuPlugin::Upscale => format!( + "clip = core.ncnn.Model(clip, network_path={}, fp16=True)", + py_str(&files.model_dir.join(UPSCALE_MODELS[0])) + ), + }; + format!( + "import sys, ctypes\n\ + def stage(s):\n print('{marker}' + s, file=sys.stderr, flush=True)\n\ + for lib in [{libs}]:\n try:\n ctypes.CDLL(lib)\n except OSError:\n stage('missing:' + lib)\n\ + stage('preload')\n\ + if {preload}:\n ctypes.CDLL({preload}, mode=ctypes.RTLD_GLOBAL)\n\ + import vapoursynth as vs\n\ + core = vs.core\n\ + stage('load')\n\ + if not hasattr(core, '{namespace}'):\n core.std.LoadPlugin({plugin_path})\n\ + stage('init')\n\ + clip = core.std.BlankClip(width=64, height=64, format=vs.RGBS, length=2, fpsnum=25, fpsden=1)\n\ + {call}\n\ + stage('render')\n\ + clip.set_output()\n", + marker = plugin.stage_marker(), + namespace = plugin.namespace(), + plugin_path = py_str(&files.plugin), + ) +} + +/// How the probe process ended. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct ProbeExit { + pub success: bool, + pub timed_out: bool, + /// Human-readable exit status for a failure (`signal 11 (SIGSEGV)` …). + pub status: String, +} + +/// Turn a finished probe into a status. Pure, so every branch is testable. +pub fn classify(plugin: GpuPlugin, exit: &ProbeExit, stderr: &str) -> GpuPluginStatus { + use GpuPluginReason as R; + if exit.success { + return GpuPluginStatus::available(); + } + if exit.timed_out { + return GpuPluginStatus::unavailable( + R::ProbeFailed, + format!("the GPU test did not finish within {}s", PROBE_TIMEOUT.as_secs()), + ); + } + + let marker = plugin.stage_marker(); + let stages: Vec<&str> = stderr + .lines() + .filter_map(|l| l.trim().strip_prefix(marker)) + .collect(); + let missing: Vec<&str> = stages.iter().filter_map(|s| s.strip_prefix("missing:")).collect(); + let last = stages.iter().rev().find(|s| !s.starts_with("missing:")).copied(); + let error = last_error_line(marker, stderr); + + // An illegal instruction is the CPU, wherever it happened. + if exit.status.starts_with("signal 4 ") || exit.status.contains("illegal instruction") { + return GpuPluginStatus::unavailable(R::UnsupportedCpu, exit.status.clone()); + } + + // RIFE's and VSNCNN's own words for "no Vulkan device". + if stderr.contains("failed to create GPU instance") + || stderr.contains("invalid GPU device") + || stderr.contains("get_gpu_device failed") + { + return GpuPluginStatus::unavailable(R::NoVulkan, error); + } + if stderr.contains("failed to load model") || stderr.contains("load param failed") { + return GpuPluginStatus::unavailable(R::NotInstalled, error); + } + match last { + // The plugin would not load. A runtime library the pre-check could not + // open is the likeliest cause, and the one the user can fix. + Some("load") if !missing.is_empty() => GpuPluginStatus::unavailable( + R::MissingLibrary, + format!("could not load {}", missing.join(", ")), + ), + Some("load") | Some("preload") => GpuPluginStatus::unavailable( + R::LoadFailed, + error.or_else(|| Some(format!("vspipe exited with {}", exit.status))), + ), + // Died creating the GPU instance without an error: no Vulkan loader or + // driver for ncnn to call into. + Some("init") => GpuPluginStatus::unavailable( + R::NoVulkan, + error.or_else(|| Some(format!("the GPU could not be initialised ({})", exit.status))), + ), + Some("render") => GpuPluginStatus::unavailable( + R::Crashed, + error.or_else(|| Some(format!("vspipe exited with {}", exit.status))), + ), + _ => GpuPluginStatus::unavailable( + R::ProbeFailed, + error.or_else(|| Some(format!("vspipe exited with {}", exit.status))), + ), + } +} + +/// The last line of a Python/VapourSynth error, which names the actual failure. +fn last_error_line(marker: &str, stderr: &str) -> Option { + stderr + .lines() + .map(str::trim) + .filter(|l| !l.is_empty() && !l.starts_with(marker) && !l.starts_with('[')) + .filter(|l| l.contains("Error") || l.contains("error") || l.contains("Failed")) + .last() + .map(|l| l.chars().take(400).collect()) +} + +/// Run the probe script through vspipe and classify the result. +pub fn run_probe( + plugin: GpuPlugin, + vspipe: &Path, + env: &HashMap, + files: &GpuPluginFiles, + platform_suffix: &str, +) -> GpuPluginStatus { + let script = probe_script(plugin, files, platform_suffix); + let tmp = std::env::temp_dir().join(format!( + "vapourbox_{}_probe_{}.vpy", + plugin.namespace(), + std::process::id() + )); + if let Err(e) = std::fs::write(&tmp, script) { + return GpuPluginStatus::unavailable( + GpuPluginReason::ProbeFailed, + format!("could not write the probe script: {e}"), + ); + } + let exit_and_stderr = run_with_timeout(vspipe, &tmp, env); + let _ = std::fs::remove_file(&tmp); + match exit_and_stderr { + Ok((exit, stderr)) => classify(plugin, &exit, &stderr), + Err(e) => GpuPluginStatus::unavailable( + GpuPluginReason::ProbeFailed, + format!("could not run vspipe: {e}"), + ), + } +} + +fn run_with_timeout( + vspipe: &Path, + script: &Path, + env: &HashMap, +) -> std::io::Result<(ProbeExit, String)> { + // Frame 1: for RIFE's 2x output the first interpolated frame (frame 0 is a + // copy and would never touch the GPU); for upscaling, any frame runs it. + let mut child = Command::new(vspipe) + .args(["-s", "1", "-e", "1"]) + .arg(script) + .arg("-") + .envs(env) + .env("MVK_CONFIG_LOG_LEVEL", "0") + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .spawn()?; + let mut stderr = child.stderr.take(); + let reader = std::thread::spawn(move || { + let mut s = String::new(); + if let Some(e) = stderr.as_mut() { + let _ = e.read_to_string(&mut s); + } + s + }); + let start = Instant::now(); + let status = loop { + if let Some(status) = child.try_wait()? { + break Some(status); + } + if start.elapsed() > PROBE_TIMEOUT { + let _ = child.kill(); + let _ = child.wait(); + break None; + } + std::thread::sleep(Duration::from_millis(50)); + }; + let stderr = reader.join().unwrap_or_default(); + let exit = match status { + Some(s) => ProbeExit { + success: s.success(), + timed_out: false, + status: crate::pipeline_executor::format_exit_status(&s), + }, + None => ProbeExit { success: false, timed_out: true, status: "timed out".into() }, + }; + Ok((exit, stderr)) +} + +#[cfg(test)] +mod tests { + use super::*; + use GpuPlugin::{Rife, Upscale}; + use GpuPluginReason as R; + + fn failed(status: &str) -> ProbeExit { + ProbeExit { success: false, timed_out: false, status: status.into() } + } + + #[test] + fn layout_per_platform() { + let mac = GpuPluginFiles::in_vapoursynth_dir(Rife, Path::new("/d/vapoursynth"), "macos-arm64"); + assert_eq!(mac.plugin, Path::new("/d/vapoursynth/rife/librife.dylib")); + assert_eq!(mac.preload.as_deref(), Some(Path::new("/d/vapoursynth/moltenvk/libMoltenVK.dylib"))); + assert!(mac.model_dir.ends_with("rife/models/rife-v4.26_ensembleFalse")); + let win = GpuPluginFiles::in_vapoursynth_dir(Rife, Path::new("D:/v"), "windows-x64"); + assert!(win.plugin.ends_with("rife/librife.dll")); + assert_eq!(win.preload, None); + let linux = GpuPluginFiles::in_vapoursynth_dir(Rife, Path::new("/v"), "linux-x64"); + assert!(linux.plugin.ends_with("rife/librife.so")); + // MoltenVK is required on macOS, and only there. + assert_eq!(mac.required().len(), 4); + assert_eq!(linux.required().len(), 3); + + let mac = GpuPluginFiles::in_vapoursynth_dir(Upscale, Path::new("/d/vapoursynth"), "macos-arm64"); + assert_eq!(mac.plugin, Path::new("/d/vapoursynth/ncnn/libvsncnn.dylib")); + assert_eq!(mac.preload.as_deref(), Some(Path::new("/d/vapoursynth/moltenvk/libMoltenVK.dylib"))); + let win = GpuPluginFiles::in_vapoursynth_dir(Upscale, Path::new("D:/v"), "windows-x64"); + assert!(win.plugin.ends_with("ncnn/vsncnn.dll")); + let linux = GpuPluginFiles::in_vapoursynth_dir(Upscale, Path::new("/v"), "linux-x64"); + assert!(linux.plugin.ends_with("ncnn/libvsncnn.so")); + assert_eq!(linux.required().len(), 1 + UPSCALE_MODELS.len()); + assert!(linux.models[0].ends_with("ncnn/models/RealESRGANv2/RealESRGANv2-animevideo-xsx2.onnx")); + } + + #[test] + fn unsupported_platforms_are_reported_before_anything_runs() { + for plugin in [Rife, Upscale] { + let files = GpuPluginFiles::in_vapoursynth_dir(plugin, Path::new("/nonexistent"), "macos-x64"); + for p in ["macos-x64", "linux-arm64", "windows-arm64"] { + let s = precheck(plugin, p, &files).expect("must decide without probing"); + assert_eq!(s.reason, Some(R::UnsupportedPlatform), "{plugin:?} {p}"); + } + // A shipping platform with no files is "not installed", not unsupported. + let files = GpuPluginFiles::in_vapoursynth_dir(plugin, Path::new("/nonexistent"), "linux-x64"); + let s = precheck(plugin, "linux-x64", &files).unwrap(); + assert_eq!(s.reason, Some(R::NotInstalled)); + } + let files = GpuPluginFiles::in_vapoursynth_dir(Rife, Path::new("/nonexistent"), "linux-x64"); + assert!(precheck(Rife, "linux-x64", &files).unwrap().detail.unwrap().contains("librife.so")); + } + + #[test] + fn every_reason_has_a_message_on_every_platform() { + let reasons = [ + R::UnsupportedPlatform, + R::NotInstalled, + R::NoVulkan, + R::MissingLibrary, + R::LoadFailed, + R::Crashed, + R::UnsupportedCpu, + R::Disabled, + R::ProbeFailed, + ]; + for plugin in [Rife, Upscale] { + for platform in ["macos-arm64", "macos-x64", "windows-x64", "linux-x64", "linux-arm64"] { + for r in reasons { + let m = GpuPluginStatus::unavailable(r, None).message(plugin, platform).unwrap(); + assert!(m.len() > 40, "{platform} {r:?}: {m}"); + assert!(m.contains(plugin.label()), "{plugin:?} {r:?}: {m}"); + } + } + assert_eq!(GpuPluginStatus::available().message(plugin, "linux-x64"), None); + } + let with_detail = GpuPluginStatus::unavailable(R::MissingLibrary, "could not load vcomp140.dll".to_string()); + let m = with_detail.message(Rife, "windows-x64").unwrap(); + assert!(m.contains("Visual C++ Redistributable") && m.contains("vcomp140.dll")); + let m = GpuPluginStatus::unavailable(R::UnsupportedPlatform, None).message(Upscale, "macos-x64").unwrap(); + assert!(m.contains("NNEDI3"), "{m}"); + } + + #[test] + fn json_carries_the_message() { + let v = GpuPluginStatus::unavailable(R::NoVulkan, None).to_json(Rife, "windows-x64"); + assert_eq!(v["available"], false); + assert_eq!(v["reason"], "noVulkan"); + assert!(v["message"].as_str().unwrap().contains("Vulkan")); + let ok = GpuPluginStatus::available().to_json(Upscale, "windows-x64"); + assert_eq!(ok, serde_json::json!({ "available": true })); + } + + #[test] + fn classify_success() { + let ok = ProbeExit { success: true, ..Default::default() }; + assert!(classify(Rife, &ok, "RIFE_PROBE:render\n").available); + } + + #[test] + fn classify_segfault_creating_the_gpu_instance_is_no_vulkan() { + // What macOS does with no MoltenVK in the process, reproduced 2026-10-24. + let s = classify(Rife, &failed("signal 11 (SIGSEGV)"), "RIFE_PROBE:preload\nRIFE_PROBE:load\nRIFE_PROBE:init\n"); + assert_eq!(s.reason, Some(R::NoVulkan)); + let s = classify(Upscale, &failed("signal 11 (SIGSEGV)"), "UPSCALE_PROBE:load\nUPSCALE_PROBE:init\n"); + assert_eq!(s.reason, Some(R::NoVulkan)); + } + + #[test] + fn classify_plugin_errors() { + let gpu = "RIFE_PROBE:init\nvapoursynth.Error: RIFE: failed to create GPU instance\n"; + assert_eq!(classify(Rife, &failed("exit code 1"), gpu).reason, Some(R::NoVulkan)); + let dev = "RIFE_PROBE:init\nvapoursynth.Error: RIFE: invalid GPU device\n"; + assert_eq!(classify(Rife, &failed("exit code 1"), dev).reason, Some(R::NoVulkan)); + let model = "RIFE_PROBE:init\nvapoursynth.Error: RIFE: failed to load model\n"; + assert_eq!(classify(Rife, &failed("exit code 1"), model).reason, Some(R::NotInstalled)); + // VSNCNN's words. + let dev = "UPSCALE_PROBE:init\nvapoursynth.Error: ncnn.Model: get_gpu_device failed\n"; + assert_eq!(classify(Upscale, &failed("exit code 1"), dev).reason, Some(R::NoVulkan)); + let param = "UPSCALE_PROBE:init\nvapoursynth.Error: ncnn.Model: load param failed\n"; + assert_eq!(classify(Upscale, &failed("exit code 1"), param).reason, Some(R::NotInstalled)); + } + + #[test] + fn classify_load_failure_names_the_missing_runtime() { + let err = "RIFE_PROBE:missing:vcomp140.dll\nRIFE_PROBE:preload\nRIFE_PROBE:load\n\ + vapoursynth.Error: Failed to load C:\\deps\\librife.dll. GetLastError() returned 126.\n"; + let s = classify(Rife, &failed("exit code 1"), err); + assert_eq!(s.reason, Some(R::MissingLibrary)); + assert!(s.detail.unwrap().contains("vcomp140.dll")); + // With every runtime present, the loader's own error is the detail. + let s = classify(Rife, &failed("exit code 1"), &err.replace("RIFE_PROBE:missing:vcomp140.dll\n", "")); + assert_eq!(s.reason, Some(R::LoadFailed)); + assert!(s.detail.unwrap().contains("GetLastError() returned 126")); + } + + #[test] + fn classify_crash_while_rendering_and_timeouts() { + let s = classify(Rife, &failed("signal 11 (SIGSEGV)"), "RIFE_PROBE:init\nRIFE_PROBE:render\n"); + assert_eq!(s.reason, Some(R::Crashed)); + assert!(s.detail.unwrap().contains("SIGSEGV")); + let t = ProbeExit { success: false, timed_out: true, status: "timed out".into() }; + assert_eq!(classify(Rife, &t, "").reason, Some(R::ProbeFailed)); + // vspipe that never reached the script at all. + assert_eq!(classify(Rife, &failed("exit code 1"), "").reason, Some(R::ProbeFailed)); + } + + #[test] + fn classify_reads_only_its_own_markers() { + // An upscale probe's stderr is not read as RIFE's, or the reverse. + let s = classify(Upscale, &failed("signal 11 (SIGSEGV)"), "RIFE_PROBE:init\nRIFE_PROBE:render\n"); + assert_eq!(s.reason, Some(R::ProbeFailed)); + } + + #[test] + fn classify_illegal_instruction_is_the_cpu() { + let sigill = classify(Rife, &failed("signal 4 (unknown)"), "RIFE_PROBE:load\n"); + assert_eq!(sigill.reason, Some(R::UnsupportedCpu)); + let win = failed("exit code -1073741795 — illegal instruction: a plugin used a CPU feature"); + assert_eq!(classify(Rife, &win, "RIFE_PROBE:load\n").reason, Some(R::UnsupportedCpu)); + } + + #[test] + fn py_str_escapes_windows_paths_and_quotes() { + assert_eq!(py_str(Path::new(r"C:\Users\o'brien\rife")), r"'C:\\Users\\o\'brien\\rife'"); + assert_eq!(py_str(Path::new("/a/b")), "'/a/b'"); + } + + #[test] + fn probe_script_preloads_only_where_a_preload_exists() { + let mac = GpuPluginFiles::in_vapoursynth_dir(Rife, Path::new("/v"), "macos-arm64"); + let s = probe_script(Rife, &mac, "macos-arm64"); + let preload = py_str(mac.preload.as_deref().unwrap()); + assert!(s.contains(&format!("ctypes.CDLL({preload}, mode=ctypes.RTLD_GLOBAL)"))); + let win = GpuPluginFiles::in_vapoursynth_dir(Rife, Path::new("C:/v"), "windows-x64"); + let s = probe_script(Rife, &win, "windows-x64"); + assert!(s.contains("if None:")); + assert!(s.contains("'vcomp140.dll'")); + // VSNCNN's Windows DLL needs no VC++ runtime, and loads by its own namespace. + let win = GpuPluginFiles::in_vapoursynth_dir(Upscale, Path::new("C:/v"), "windows-x64"); + let s = probe_script(Upscale, &win, "windows-x64"); + assert!(!s.contains("vcomp140")); + assert!(s.contains("hasattr(core, 'ncnn')") && s.contains("core.ncnn.Model(")); + } +} diff --git a/worker/src/lib.rs b/worker/src/lib.rs index 96955dc8..c67fa74a 100644 --- a/worker/src/lib.rs +++ b/worker/src/lib.rs @@ -12,7 +12,8 @@ pub mod pipeline_executor; pub mod pixel_format; pub mod source_decode; pub mod progress_reporter; -pub mod rife; +pub mod gpu_plugin; +pub mod native_res; pub mod schema_script_generator; pub mod script_generator; pub mod subtitle_generator; diff --git a/worker/src/main.rs b/worker/src/main.rs index 0583677c..fe07df17 100644 --- a/worker/src/main.rs +++ b/worker/src/main.rs @@ -26,7 +26,8 @@ mod pipeline_executor; mod pixel_format; mod source_decode; mod progress_reporter; -mod rife; +mod gpu_plugin; +mod native_res; mod script_generator; mod subtitle_generator; mod platform; @@ -99,6 +100,16 @@ struct Args { /// what to do about it. #[arg(long)] probe_rife: bool, + + /// Probe whether AI upscaling (Real-ESRGAN / Real-CUGAN through VSNCNN) + /// can run here; prints the same JSON shape as `--probe-rife`. + #[arg(long)] + probe_upscale: bool, + + /// Detect the resolution and kernel the video in `--config` was upscaled + /// from (for the Descale pass); prints a JSON report. + #[arg(long)] + detect_native: bool, } fn main() -> ExitCode { @@ -149,7 +160,15 @@ fn main() -> ExitCode { } if args.probe_rife { - return run_probe_rife(); + return run_probe_gpu_plugin(gpu_plugin::GpuPlugin::Rife); + } + + if args.probe_upscale { + return run_probe_gpu_plugin(gpu_plugin::GpuPlugin::Upscale); + } + + if args.detect_native { + return run_detect_native(&args); } // Preview mode outputs raw PNG to stdout - no JSON messages @@ -210,22 +229,71 @@ fn run_probe_opencl() -> ExitCode { // `knlm` gates the knlmeanscl denoiser (a distinct probe — KNLMeansCL can // fail where NNEDI3CL succeeds). Both fall back to false if deps can't be // located, so the app warns rather than letting a job fail cryptically. - let (opencl, knlm) = dependency_locator::DependencyLocator::new() - .map(|deps| (deps.opencl_available(), deps.knlm_available())) - .unwrap_or((false, false)); - println!("{}", serde_json::json!({ "opencl": opencl, "knlm": knlm })); + // `knlmDevice` is the device the KNLMeans noise-reduction method runs on + // (null when none works, with `knlmMessage` saying why) — not the same + // question as `knlm`, which is about havsfunc's default-device call. + let (opencl, knlm, knlm_runtime) = match dependency_locator::DependencyLocator::new() { + Ok(deps) => (deps.opencl_available(), deps.knlm_available(), deps.knlm_runtime()), + Err(_) => ( + false, + false, + Err(dependency_locator::KNLM_NO_DEVICE_MESSAGE.to_string()), + ), + }; + let (knlm_device, knlm_message) = match knlm_runtime { + Ok(device) => (Some(device), None), + Err(message) => (None, Some(message)), + }; + println!( + "{}", + serde_json::json!({ + "opencl": opencl, + "knlm": knlm, + "knlmDevice": knlm_device, + "knlmMessage": knlm_message, + }) + ); ExitCode::SUCCESS } -/// Probe whether RIFE runs here and print the result as JSON to stdout. Deps -/// that cannot be located are reported as "not installed" rather than as a -/// failure of the probe: the remedy is the same, let the app download them. -fn run_probe_rife() -> ExitCode { +/// Detect a source's native resolution and kernel and print the report as one +/// JSON line (`{"error": ...}` when it could not be analysed). +fn run_detect_native(args: &Args) -> ExitCode { + let result = (|| -> anyhow::Result { + let config = args + .config + .as_ref() + .ok_or_else(|| anyhow::anyhow!("--config is required with --detect-native"))?; + let mut job: models::VideoJob = serde_json::from_str(&std::fs::read_to_string(config)?)?; + let deps = dependency_locator::DependencyLocator::new()?; + fill_frame_size(&mut job, Some(&deps), |_| {}); + native_res::detect(&deps, &job) + })(); + match result { + Ok(report) => { + println!("{}", serde_json::to_string(&report).unwrap_or_default()); + ExitCode::SUCCESS + } + Err(e) => { + println!("{}", serde_json::json!({ "error": e.to_string() })); + ExitCode::FAILURE + } + } +} + +/// Probe whether a GPU plugin (RIFE, AI upscaling) runs here and print the +/// result as JSON to stdout. Deps that cannot be located are reported as "not +/// installed" rather than as a failure of the probe: the remedy is the same, +/// let the app download them. +fn run_probe_gpu_plugin(plugin: gpu_plugin::GpuPlugin) -> ExitCode { let json = match dependency_locator::DependencyLocator::new() { - Ok(deps) => deps.rife_status().to_json(deps.platform_suffix()), - Err(e) => rife::RifeStatus::unavailable(rife::RifeReason::NotInstalled, e.to_string()) - .to_json(""), + Ok(deps) => deps.gpu_plugin_status(plugin).to_json(plugin, deps.platform_suffix()), + Err(e) => gpu_plugin::GpuPluginStatus::unavailable( + gpu_plugin::GpuPluginReason::NotInstalled, + e.to_string(), + ) + .to_json(plugin, ""), }; println!("{json}"); ExitCode::SUCCESS @@ -559,19 +627,38 @@ fn run_worker( // Detect knlmeanscl availability separately (plugin + usable OpenCL device); // when unavailable the knlmeanscl denoiser is downgraded to dfttest. let knlm_available = deps.as_ref().map(|d| d.knlm_available()).unwrap_or(true); - // RIFE is probed only for a job that uses it: the probe initialises the GPU. - let rife = if job.effective_pipeline().uses_rife() { + // The GPU plugins are probed only for a job that uses them: the probe + // initialises the GPU. + let gpu_runtime = |plugin: gpu_plugin::GpuPlugin, used: bool| { + if !used { + return Err(format!("{} is not used by this job.", plugin.label())); + } + match deps.as_ref() { + Some(d) => d.gpu_plugin_runtime(plugin), + None => Err(format!( + "{} needs VapourBox's dependencies, which could not be found.", + plugin.label() + )), + } + }; + let pipeline = job.effective_pipeline(); + let rife = gpu_runtime(gpu_plugin::GpuPlugin::Rife, pipeline.uses_rife()); + let upscale = gpu_runtime(gpu_plugin::GpuPlugin::Upscale, pipeline.uses_ai_upscale()); + // Likewise KNLMeans, whose device probe runs KNLMeansCL once per boot. + let knlm = if job.effective_pipeline().uses_knlm() { match deps.as_ref() { - Some(d) => d.rife_runtime(), - None => Err("RIFE needs VapourBox's dependencies, which could not be found.".into()), + Some(d) => d.knlm_runtime(), + None => Err("KNLMeans needs VapourBox's dependencies, which could not be found.".into()), } } else { - Err("RIFE is not used by this job.".into()) + Err("KNLMeans is not used by this job.".into()) }; let script_generator = ScriptGenerator::new()? .with_opencl_available(opencl_available) .with_knlm_available(knlm_available) - .with_rife(rife); + .with_rife(rife) + .with_upscale(upscale) + .with_knlm(knlm); let script_path = script_generator .generate(&job) .with_context(|| "Failed to generate VapourSynth script")?; diff --git a/worker/src/models/crop_resize_parameters.rs b/worker/src/models/crop_resize_parameters.rs index 5478bf5a..b51b1493 100644 --- a/worker/src/models/crop_resize_parameters.rs +++ b/worker/src/models/crop_resize_parameters.rs @@ -67,6 +67,18 @@ pub enum UpscaleMethod { Nnedi3Rpow2, Eedi3Rpow2, Spline36, + /// Real-ESRGAN's anime video model (BSD-3), on the GPU through VSNCNN. + RealEsrgan, + /// Real-CUGAN (MIT, bilibili), on the GPU through VSNCNN. + RealCugan, +} + +impl UpscaleMethod { + /// Whether this is one of the neural upscalers that need the VSNCNN plugin + /// and a Vulkan device (see `gpu_plugin.rs`). + pub fn is_ai(self) -> bool { + matches!(self, UpscaleMethod::RealEsrgan | UpscaleMethod::RealCugan) + } } /// What to do with the pixel aspect ratio (SAR) on output. @@ -246,6 +258,11 @@ pub struct CropResizeParameters { #[serde(default = "default_upscale_factor")] pub upscale_factor: i32, + /// Real-CUGAN's denoise strength: "none", "conservative" or "denoise3x" — + /// the three bundled 2x models. + #[serde(default = "default_cugan_noise")] + pub cugan_noise: String, + // --- nnedi3 controls for the edge-directed upscale paths --- // With EEDI3 selected these shape the nnedi3 sclip that guides it. @@ -294,6 +311,7 @@ pub struct CropResizeParameters { fn default_true() -> bool { true } fn default_upscale_factor() -> i32 { 2 } +fn default_cugan_noise() -> String { "conservative".to_string() } impl Default for CropResizeParameters { fn default() -> Self { @@ -325,6 +343,7 @@ impl Default for CropResizeParameters { use_integer_upscale: false, upscale_method: UpscaleMethod::default(), upscale_factor: default_upscale_factor(), + cugan_noise: default_cugan_noise(), upscale_nsize: None, upscale_neurons: None, upscale_qual: None, @@ -379,6 +398,41 @@ pub enum AspectDeclaration { #[allow(dead_code)] impl CropResizeParameters { + /// Whether this pass upscales with a neural model, which needs the VSNCNN + /// plugin and a working Vulkan device. + pub fn uses_ai_upscale(&self) -> bool { + self.enabled && self.use_integer_upscale && self.upscale_method.is_ai() + } + + /// The models an AI upscale runs, in order, as paths under + /// `vapoursynth/ncnn/models/`. Factors follow the edge-directed path's + /// rule (3x behaves as 2x). Real-ESRGAN has a 4x model; Real-CUGAN's + /// bundled models are 2x only, so 4x runs it twice, denoising on the first + /// pass only — a second denoise pass would smooth the result twice. + pub fn ai_upscale_models(&self) -> Vec<&'static str> { + let four = self.upscale_factor >= 4; + match self.upscale_method { + UpscaleMethod::RealEsrgan => vec![if four { + "RealESRGANv2/RealESRGANv2-animevideo-xsx4.onnx" + } else { + "RealESRGANv2/RealESRGANv2-animevideo-xsx2.onnx" + }], + UpscaleMethod::RealCugan => { + let first = match self.cugan_noise.as_str() { + "none" => "cugan/up2x-latest-no-denoise.onnx", + "denoise3x" => "cugan/up2x-latest-denoise3x.onnx", + _ => "cugan/up2x-latest-conservative.onnx", + }; + if four { + vec![first, "cugan/up2x-latest-no-denoise.onnx"] + } else { + vec![first] + } + } + _ => Vec::new(), + } + } + /// Whether the pixel grid itself has to be resampled to square up the /// pixels. True even with no target size — "make this anamorphic source /// square" is a resize in its own right. @@ -669,4 +723,41 @@ mod tests { assert!(json.contains("\"enabled\":false")); assert!(json.contains("\"maintainAspect\":true")); } + + #[test] + fn test_ai_upscale_models_follow_factor_and_noise() { + let p = |method, factor, noise: &str| CropResizeParameters { + enabled: true, + use_integer_upscale: true, + upscale_method: method, + upscale_factor: factor, + cugan_noise: noise.into(), + ..Default::default() + }; + assert_eq!(p(UpscaleMethod::RealEsrgan, 2, "").ai_upscale_models(), + ["RealESRGANv2/RealESRGANv2-animevideo-xsx2.onnx"]); + assert_eq!(p(UpscaleMethod::RealEsrgan, 3, "").ai_upscale_models(), + ["RealESRGANv2/RealESRGANv2-animevideo-xsx2.onnx"]); + assert_eq!(p(UpscaleMethod::RealEsrgan, 4, "").ai_upscale_models(), + ["RealESRGANv2/RealESRGANv2-animevideo-xsx4.onnx"]); + assert_eq!(p(UpscaleMethod::RealCugan, 2, "denoise3x").ai_upscale_models(), + ["cugan/up2x-latest-denoise3x.onnx"]); + assert_eq!(p(UpscaleMethod::RealCugan, 4, "none").ai_upscale_models(), + ["cugan/up2x-latest-no-denoise.onnx", "cugan/up2x-latest-no-denoise.onnx"]); + // An unknown strength from a hand-edited preset is the default. + assert_eq!(p(UpscaleMethod::RealCugan, 2, "max").ai_upscale_models(), + ["cugan/up2x-latest-conservative.onnx"]); + // Every model named here is one the deps bundle ships. + for m in [UpscaleMethod::RealEsrgan, UpscaleMethod::RealCugan] { + for f in [2, 4] { + for n in ["none", "conservative", "denoise3x"] { + for model in p(m, f, n).ai_upscale_models() { + assert!(crate::gpu_plugin::UPSCALE_MODELS.contains(&model), "{model}"); + } + } + } + } + assert!(!p(UpscaleMethod::Nnedi3Rpow2, 2, "").uses_ai_upscale()); + assert!(p(UpscaleMethod::RealCugan, 2, "").uses_ai_upscale()); + } } diff --git a/worker/src/models/descale_parameters.rs b/worker/src/models/descale_parameters.rs new file mode 100644 index 00000000..aaaaed80 --- /dev/null +++ b/worker/src/models/descale_parameters.rs @@ -0,0 +1,179 @@ +//! Descale: undo a studio's upscale, then upscale again properly. +//! +//! Most modern anime is drawn below 1080p and upscaled for broadcast or Blu-ray +//! with a plain resampling kernel. When the native resolution and kernel are +//! known, `descale` inverts that resample exactly, recovering the picture as it +//! was drawn; NNEDI3 then upscales it back to the source size with far cleaner +//! lines than the studio's kernel gave. Luma only — chroma was stored at a +//! lower resolution to begin with — and the output stays at the source size. + +use serde::{Deserialize, Serialize}; + +/// The resampling kernel the studio upscaled with. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +pub enum DescaleKernel { + Bilinear, + /// Bicubic with `b`/`c` (Catmull-Rom b=0 c=0.5 is the usual one). + #[default] + Bicubic, + Lanczos, + Spline16, + Spline36, + Spline64, +} + +/// How the studio's resampler treated pixels past the frame edge. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +pub enum DescaleBorder { + #[default] + Mirror, + Zero, + Extend, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct DescaleParameters { + #[serde(default)] + pub enabled: bool, + + /// The height the picture was drawn at. + #[serde(default = "default_native_height")] + pub native_height: i32, + + /// The width it was drawn at; 0 derives it from the source's shape. + #[serde(default)] + pub native_width: i32, + + #[serde(default)] + pub kernel: DescaleKernel, + + #[serde(default)] + pub bicubic_b: f64, + + #[serde(default = "default_bicubic_c")] + pub bicubic_c: f64, + + #[serde(default = "default_lanczos_taps")] + pub lanczos_taps: i32, + + #[serde(default)] + pub border: DescaleBorder, + + /// Keep the source wherever re-applying the kernel to the descaled picture + /// does not reproduce it: detail drawn at full resolution (credits, text, + /// overlays), which descaling would otherwise smear. + #[serde(default = "default_true")] + pub protect_detail: bool, + + /// How far (8-bit levels) the re-applied kernel may miss before a pixel + /// counts as full-resolution detail. + #[serde(default = "default_detail_threshold")] + pub detail_threshold: f64, +} + +fn default_native_height() -> i32 { 720 } +fn default_bicubic_c() -> f64 { 0.5 } +fn default_lanczos_taps() -> i32 { 3 } +fn default_true() -> bool { true } +fn default_detail_threshold() -> f64 { 4.0 } + +impl Default for DescaleParameters { + fn default() -> Self { + Self { + enabled: false, + native_height: default_native_height(), + native_width: 0, + kernel: DescaleKernel::default(), + bicubic_b: 0.0, + bicubic_c: default_bicubic_c(), + lanczos_taps: default_lanczos_taps(), + border: DescaleBorder::default(), + protect_detail: true, + detail_threshold: default_detail_threshold(), + } + } +} + +impl DescaleParameters { + /// The descale call for a GRAY float clip `{clip}` at `{w}`x`{h}`, as + /// Python — and the matching forward resize that re-applies the kernel. + pub fn kernel_calls(&self) -> (String, String) { + let border = match self.border { + DescaleBorder::Mirror => 0, + DescaleBorder::Zero => 1, + DescaleBorder::Extend => 2, + }; + let taps = self.lanczos_taps.clamp(1, 8); + let (b, c) = (self.bicubic_b.clamp(-1.0, 2.0), self.bicubic_c.clamp(-1.0, 2.0)); + let (de, fwd) = match self.kernel { + DescaleKernel::Bilinear => ( + "core.descale.Debilinear(c, w, h".to_string(), + "core.resize.Bilinear(c, w, h".to_string(), + ), + DescaleKernel::Bicubic => ( + format!("core.descale.Debicubic(c, w, h, b={b}, c={c}"), + format!("core.resize.Bicubic(c, w, h, filter_param_a={b}, filter_param_b={c}"), + ), + DescaleKernel::Lanczos => ( + format!("core.descale.Delanczos(c, w, h, taps={taps}"), + format!("core.resize.Lanczos(c, w, h, filter_param_a={taps}"), + ), + DescaleKernel::Spline16 => ( + "core.descale.Despline16(c, w, h".to_string(), + "core.resize.Spline16(c, w, h".to_string(), + ), + DescaleKernel::Spline36 => ( + "core.descale.Despline36(c, w, h".to_string(), + "core.resize.Spline36(c, w, h".to_string(), + ), + DescaleKernel::Spline64 => ( + "core.descale.Despline64(c, w, h".to_string(), + "core.resize.Spline64(c, w, h".to_string(), + ), + }; + (format!("{de}, border_handling={border})"), format!("{fwd})")) + } + + /// The native height, kept to something descale can work with. + pub fn effective_native_height(&self) -> i32 { + self.native_height.clamp(16, 4320) + } + + /// The detail threshold, clamped to a sensible 0.5-64 levels. + pub fn effective_detail_threshold(&self) -> f64 { + self.detail_threshold.clamp(0.5, 64.0) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn kernel_calls_pair_the_inverse_with_its_forward_resize() { + let p = DescaleParameters { kernel: DescaleKernel::Bicubic, bicubic_b: 0.0, bicubic_c: 0.5, ..Default::default() }; + let (de, fwd) = p.kernel_calls(); + assert_eq!(de, "core.descale.Debicubic(c, w, h, b=0, c=0.5, border_handling=0)"); + assert_eq!(fwd, "core.resize.Bicubic(c, w, h, filter_param_a=0, filter_param_b=0.5)"); + let p = DescaleParameters { kernel: DescaleKernel::Lanczos, lanczos_taps: 40, border: DescaleBorder::Extend, ..Default::default() }; + let (de, fwd) = p.kernel_calls(); + assert_eq!(de, "core.descale.Delanczos(c, w, h, taps=8, border_handling=2)"); + assert_eq!(fwd, "core.resize.Lanczos(c, w, h, filter_param_a=8)"); + for k in [DescaleKernel::Bilinear, DescaleKernel::Spline16, DescaleKernel::Spline36, DescaleKernel::Spline64] { + let (de, fwd) = DescaleParameters { kernel: k, ..Default::default() }.kernel_calls(); + assert!(de.starts_with("core.descale.De") && fwd.starts_with("core.resize."), "{k:?}"); + } + } + + #[test] + fn wire_names() { + let json = serde_json::to_string(&DescaleParameters::default()).unwrap(); + assert!(json.contains("\"kernel\":\"bicubic\"")); + assert!(json.contains("\"nativeHeight\":720")); + assert!(json.contains("\"border\":\"mirror\"")); + assert!(json.contains("\"protectDetail\":true")); + } +} diff --git a/worker/src/models/frame_rate_parameters.rs b/worker/src/models/frame_rate_parameters.rs index 79923c43..1243b140 100644 --- a/worker/src/models/frame_rate_parameters.rs +++ b/worker/src/models/frame_rate_parameters.rs @@ -72,7 +72,7 @@ pub enum FrameRateMethod { /// RIFE 4.26 (ncnn/Vulkan): a neural flow estimator, on the GPU. Measured /// 3.73 mean abs diff against the real dropped frame on `small_clip` (25 → 50), /// against FlowFPS's 4.09, at roughly a fifth of its speed. It needs a - /// Vulkan device, so it is only offered where `RifeStatus` says it runs. + /// Vulkan device, so it is only offered where `GpuPluginStatus` says it runs. #[serde(rename = "rife")] Rife, } diff --git a/worker/src/models/mod.rs b/worker/src/models/mod.rs index 982f1281..3300ac4d 100644 --- a/worker/src/models/mod.rs +++ b/worker/src/models/mod.rs @@ -21,6 +21,7 @@ mod ghost_removal_parameters; mod deblock_parameters; mod deband_parameters; mod descratch_parameters; +mod descale_parameters; mod spotless_parameters; mod stabilize_parameters; mod sharpen_parameters; @@ -47,6 +48,7 @@ pub use ghost_removal_parameters::*; pub use deblock_parameters::*; pub use deband_parameters::*; pub use descratch_parameters::*; +pub use descale_parameters::*; pub use stabilize_parameters::*; pub use spotless_parameters::*; pub use sharpen_parameters::*; diff --git a/worker/src/models/noise_reduction_parameters.rs b/worker/src/models/noise_reduction_parameters.rs index 4df57b8f..9e869475 100644 --- a/worker/src/models/noise_reduction_parameters.rs +++ b/worker/src/models/noise_reduction_parameters.rs @@ -37,6 +37,15 @@ pub enum NoiseReductionMethod { /// TemporalDegrain2: the heavyweight multi-pass motion-compensated /// denoiser. Slow (about 21 fps) and the most capable thing here. TemporalDegrain2, + /// BM3D (block-matching 3D). Groups similar blocks across the frame (and, + /// with a radius, across frames) and filters each group together, so flat + /// areas come out clean without smearing edges — why anime encoders favour + /// it. CPU, ~14 fps at SD with the defaults. + Bm3d, + /// KNLMeansCL: non-local means on the GPU (OpenCL). Fast where a device + /// works; refused with a reason where none does (see + /// `DependencyLocator::knlm_device`). + KnlMeans, } /// Noise reduction preset levels. @@ -269,6 +278,52 @@ pub struct NoiseReductionParameters { /// Planes to filter: 0 luma only, 1 chroma only, 2 both. #[serde(default = "default_ctmf_planes")] pub ctmf_planes: i32, + + // --- BM3D Parameters --- + + /// Luma strength, in 8-bit units at every depth (the plugin normalises it). + #[serde(default = "default_bm3d_sigma")] + pub bm3d_sigma: f64, + + /// Chroma strength; 0 leaves chroma alone. + #[serde(default)] + pub bm3d_chroma_sigma: f64, + + /// Temporal radius in frames each side; 0 is spatial-only BM3D, more uses + /// V-BM3D across 2r+1 frames. + #[serde(default = "default_bm3d_radius")] + pub bm3d_radius: i32, + + /// Run the second (Wiener) estimate on top of the basic one: roughly half + /// the speed for noticeably cleaner flat areas. + #[serde(default = "default_true")] + pub bm3d_final: bool, + + /// The plugin's speed/quality profile: fast, lc, np (its default), high, vn. + #[serde(default = "default_bm3d_profile")] + pub bm3d_profile: String, + + // --- KNLMeansCL Parameters --- + + /// Strength. + #[serde(default = "default_knlm_h")] + pub knlm_h: f64, + + /// Temporal radius in frames each side; 0 is spatial-only. + #[serde(default = "default_knlm_d")] + pub knlm_d: i32, + + /// Search window radius. + #[serde(default = "default_knlm_a")] + pub knlm_a: i32, + + /// Similarity neighbourhood radius. + #[serde(default = "default_knlm_s")] + pub knlm_s: i32, + + /// Denoise chroma too, in a second pass (4:2:0 cannot take `channels="YUV"`). + #[serde(default = "default_true")] + pub knlm_chroma: bool, } fn default_sm_degrain_tr() -> i32 { 2 } @@ -298,6 +353,13 @@ fn default_flux_spatial() -> i32 { 7 } fn default_stpresso_limit() -> i32 { 3 } fn default_stpresso_bias() -> i32 { 24 } fn default_stpresso_tthr() -> i32 { 12 } +fn default_bm3d_sigma() -> f64 { 3.0 } +fn default_bm3d_radius() -> i32 { 1 } +fn default_bm3d_profile() -> String { "np".to_string() } +fn default_knlm_h() -> f64 { 1.2 } +fn default_knlm_d() -> i32 { 1 } +fn default_knlm_a() -> i32 { 2 } +fn default_knlm_s() -> i32 { 4 } fn default_ctmf_radius() -> i32 { 2 } fn default_ctmf_planes() -> i32 { 2 } @@ -363,6 +425,16 @@ impl Default for NoiseReductionParameters { stpresso_tthr: default_stpresso_tthr(), ctmf_radius: default_ctmf_radius(), ctmf_planes: default_ctmf_planes(), + bm3d_sigma: default_bm3d_sigma(), + bm3d_chroma_sigma: 0.0, + bm3d_radius: default_bm3d_radius(), + bm3d_final: true, + bm3d_profile: default_bm3d_profile(), + knlm_h: default_knlm_h(), + knlm_d: default_knlm_d(), + knlm_a: default_knlm_a(), + knlm_s: default_knlm_s(), + knlm_chroma: true, } } } @@ -397,6 +469,52 @@ impl NoiseReductionParameters { } } + /// Frames each side the method reads, for the preview's decode window. 3 + /// covers every motion-compensated method; BM3D's Final step re-reads its + /// basic estimate's neighbours, so it reaches twice its radius. + pub fn temporal_radius(&self) -> u32 { + let own = match self.method { + NoiseReductionMethod::Bm3d => { + let r = self.bm3d_effective_radius(); + if self.bm3d_final { 2 * r } else { r } + } + NoiseReductionMethod::KnlMeans => self.knlm_effective().1, + _ => 0, + }; + own.max(3) as u32 + } + + /// BM3D's luma and chroma strengths, clamped to 0-50 (8-bit units). + pub fn bm3d_effective_sigmas(&self) -> (f64, f64) { + (self.bm3d_sigma.clamp(0.0, 50.0), self.bm3d_chroma_sigma.clamp(0.0, 50.0)) + } + + /// BM3D's temporal radius, clamped to 0-3: each step costs about a third + /// more time, and past 3 the frames are too far apart to match. + pub fn bm3d_effective_radius(&self) -> i32 { + self.bm3d_radius.clamp(0, 3) + } + + /// A profile the plugin knows, else its own default `np`. An unknown name + /// is a hard error in the plugin, so a hand-edited preset falls back here. + pub fn bm3d_effective_profile(&self) -> &str { + match self.bm3d_profile.as_str() { + p @ ("fast" | "lc" | "np" | "high" | "vn") => p, + _ => "np", + } + } + + /// KNLMeansCL's (h, d, a, s), clamped to what it accepts and what is + /// sensible: h > 0 (0 is rejected), radii within the plugin's limits. + pub fn knlm_effective(&self) -> (f64, i32, i32, i32) { + ( + self.knlm_h.clamp(0.1, 20.0), + self.knlm_d.clamp(0, 4), + self.knlm_a.clamp(1, 4), + self.knlm_s.clamp(0, 4), + ) + } + /// Degrain frames, clamped to the 1-3 mvtools provides. pub fn mcds_effective_frames(&self) -> i32 { self.mcds_frames.clamp(1, 3) @@ -566,6 +684,8 @@ mod tests { (NoiseReductionMethod::FluxSmoothSt, "fluxSmoothSt"), (NoiseReductionMethod::StPresso, "stPresso"), (NoiseReductionMethod::Ctmf, "ctmf"), + (NoiseReductionMethod::Bm3d, "bm3d"), + (NoiseReductionMethod::KnlMeans, "knlMeans"), ] { let json = name(method); assert!( @@ -653,4 +773,37 @@ mod tests { assert_eq!(with_maxr(3).ttemp_effective_maxr(), 3); assert_eq!(with_maxr(12).ttemp_effective_maxr(), 7); } + + #[test] + fn test_bm3d_and_knlm_values_are_clamped_to_what_the_plugins_take() { + let p = NoiseReductionParameters { + bm3d_sigma: -2.0, + bm3d_chroma_sigma: 99.0, + bm3d_radius: 9, + bm3d_profile: "turbo".into(), + knlm_h: 0.0, + knlm_d: 7, + knlm_a: 0, + knlm_s: -1, + ..Default::default() + }; + assert_eq!(p.bm3d_effective_sigmas(), (0.0, 50.0)); + assert_eq!(p.bm3d_effective_radius(), 3); + assert_eq!(p.bm3d_effective_profile(), "np"); + assert_eq!(p.knlm_effective(), (0.1, 4, 1, 0)); + let d = NoiseReductionParameters::default(); + assert_eq!(d.bm3d_effective_profile(), "np"); + assert_eq!(d.bm3d_effective_radius(), 1); + } + + #[test] + fn test_temporal_radius_covers_bm3d_final_and_knlm() { + let with = |m, r, f, d| NoiseReductionParameters { + method: m, bm3d_radius: r, bm3d_final: f, knlm_d: d, ..Default::default() + }; + assert_eq!(with(NoiseReductionMethod::SmDegrain, 3, true, 4).temporal_radius(), 3); + assert_eq!(with(NoiseReductionMethod::Bm3d, 3, true, 0).temporal_radius(), 6); + assert_eq!(with(NoiseReductionMethod::Bm3d, 3, false, 0).temporal_radius(), 3); + assert_eq!(with(NoiseReductionMethod::KnlMeans, 0, true, 4).temporal_radius(), 4); + } } diff --git a/worker/src/models/processing_pipeline.rs b/worker/src/models/processing_pipeline.rs index f172b457..c07371db 100644 --- a/worker/src/models/processing_pipeline.rs +++ b/worker/src/models/processing_pipeline.rs @@ -6,7 +6,7 @@ use super::{ AntiAliasParameters, GeometryParameters, GrainParameters, StabilizeParameters, ChromaDenoiseParameters, ChromaFixParameters, ColorCorrectionParameters, CropResizeParameters, DebandParameters, DeblockParameters, DehaloParameters, DeinterlaceMethod, DeScratchParameters, - SpotLessParameters, SharpenParameters, NoiseReductionParameters, QTGMCParameters, FrameRateParameters, FrameRateMethod, DeflickerParameters, EdgeRepairParameters, GhostRemovalParameters,}; + SpotLessParameters, SharpenParameters, NoiseReductionParameters, QTGMCParameters, FrameRateParameters, FrameRateMethod, NoiseReductionMethod, DeflickerParameters, EdgeRepairParameters, GhostRemovalParameters, DescaleParameters,}; /// Minimum temporal context (frames on each side of the target) a windowed /// preview decodes, regardless of which filters are enabled. Motion-compensated @@ -116,6 +116,8 @@ pub enum PassType { Deinterlace, DeScratch, SpotLess, + /// Undo a studio's upscale and upscale again properly. + Descale, NoiseReduction, ChromaDenoise, Dehalo, @@ -159,6 +161,7 @@ impl PassType { PassType::Grain => "Film Grain", PassType::Deflicker => "Deflicker", PassType::GhostRemoval => "Ghost Removal", + PassType::Descale => "Descale", PassType::EdgeRepair => "Edge Repair", PassType::FrameRate => "Frame Rate", PassType::ColorCorrection => "Color Correction", @@ -185,6 +188,7 @@ impl PassType { PassType::Grain => "Add film grain back after denoising", PassType::Deflicker => "Even out brightness pulsing between frames", PassType::GhostRemoval => "Remove the displaced echo RF and cable distribution leave behind", + PassType::Descale => "Undo a studio's upscale, then upscale again cleanly", PassType::EdgeRepair => "Rebuild the dirty rows and columns at the frame border", PassType::FrameRate => "Convert between PAL and NTSC frame rates", PassType::ColorCorrection => "Adjust brightness, contrast, and colors", @@ -258,6 +262,9 @@ pub struct ProcessingPipeline { #[serde(default)] pub ghost_removal: GhostRemovalParameters, + #[serde(default)] + pub descale: DescaleParameters, + #[serde(default)] pub edge_repair: EdgeRepairParameters, @@ -296,6 +303,7 @@ impl Default for ProcessingPipeline { grain: GrainParameters::default(), deflicker: DeflickerParameters::default(), ghost_removal: GhostRemovalParameters::default(), + descale: DescaleParameters::default(), edge_repair: EdgeRepairParameters::default(), frame_rate: FrameRateParameters::default(), color_correction: ColorCorrectionParameters::default(), @@ -325,6 +333,7 @@ impl ProcessingPipeline { grain: GrainParameters { enabled: false, ..Default::default() }, deflicker: DeflickerParameters { enabled: false, ..Default::default() }, ghost_removal: GhostRemovalParameters { enabled: false, ..Default::default() }, + descale: DescaleParameters::default(), edge_repair: EdgeRepairParameters { enabled: false, ..Default::default() }, frame_rate: FrameRateParameters { enabled: false, ..Default::default() }, color_correction: ColorCorrectionParameters { enabled: false, ..Default::default() }, @@ -367,6 +376,11 @@ impl ProcessingPipeline { if self.spotless.enabled { passes.push(PassType::SpotLess); } + // Descale needs the studio's upscale exactly as it was: before any + // denoise, deblock or sharpen changes the picture it inverts. + if self.descale.enabled { + passes.push(PassType::Descale); + } if self.noise_reduction.enabled { passes.push(PassType::NoiseReduction); } @@ -463,15 +477,28 @@ impl ProcessingPipeline { count } - /// The frame-timeline map each enabled pass imposes, in application order. - /// Only the deinterlace pass changes the count today; the rest are identity - /// (with a small temporal radius for context-sensitive filters). - /// Whether this pipeline interpolates with RIFE — the one pass that needs a - /// GPU probe before its script can be generated. + /// Whether this pipeline interpolates with RIFE, which needs a GPU probe + /// before its script can be generated. pub fn uses_rife(&self) -> bool { self.frame_rate.enabled && self.frame_rate.method == FrameRateMethod::Rife } + /// Whether this pipeline upscales with a neural model (Real-ESRGAN / + /// Real-CUGAN), which needs a GPU probe before its script can be generated. + pub fn uses_ai_upscale(&self) -> bool { + self.crop_resize.uses_ai_upscale() + } + + /// Whether this pipeline denoises with KNLMeans, which needs a working + /// OpenCL device before its script can be generated. + pub fn uses_knlm(&self) -> bool { + self.noise_reduction.enabled + && self.noise_reduction.method == NoiseReductionMethod::KnlMeans + } + + /// The frame-timeline map each enabled pass imposes, in application order. + /// Only the deinterlace pass changes the count today; the rest are identity + /// (with a small temporal radius for context-sensitive filters). pub fn frame_maps(&self) -> Vec { self.enabled_passes() .into_iter() @@ -485,7 +512,9 @@ impl ProcessingPipeline { PassType::Deinterlace => self.deinterlace_frame_map(), // Temporal filters need neighbours for a correct windowed preview, // but don't change the frame count. - PassType::NoiseReduction => FrameMap::Identity { radius: 3 }, + PassType::NoiseReduction => FrameMap::Identity { + radius: self.noise_reduction.temporal_radius(), + }, // CCD is spatial at temporal_radius 0, temporal above it. PassType::ChromaDenoise => FrameMap::Identity { radius: self.chroma_denoise.radius(), @@ -495,6 +524,7 @@ impl ProcessingPipeline { PassType::Deflicker => FrameMap::Identity { radius: self.deflicker.radius() }, PassType::EdgeRepair => FrameMap::Identity { radius: 0 }, PassType::GhostRemoval => FrameMap::Identity { radius: 0 }, + PassType::Descale => FrameMap::Identity { radius: 0 }, // The one pass that emits a Retime. The ratio is known up front, so // the count is exact; `synthesizes` is true only for the // interpolating method, where an output frame has no single source. @@ -610,6 +640,7 @@ impl ProcessingPipeline { PassType::Grain => self.grain.has_effect(), PassType::Deflicker => self.deflicker.enabled, PassType::GhostRemoval => self.ghost_removal.has_effect(), + PassType::Descale => self.descale.enabled, PassType::EdgeRepair => self.edge_repair.has_effect(), PassType::FrameRate => self.frame_rate.enabled, PassType::ColorCorrection => self.color_correction.enabled, diff --git a/worker/src/native_res.rs b/worker/src/native_res.rs new file mode 100644 index 00000000..062f4b13 --- /dev/null +++ b/worker/src/native_res.rs @@ -0,0 +1,329 @@ +//! Detect the resolution and kernel a source was upscaled from — what the +//! Descale pass needs to know, and what a user cannot see by looking. +//! +//! The method is the one getnative made standard: descale the luma to each +//! candidate height and resample it back with the same kernel. At the height +//! the picture was drawn at, with the kernel the studio used, the round trip +//! reproduces the source almost exactly; everywhere else it loses detail. So +//! the error, plotted against height, shows a sharp dip at the native height — +//! and among kernels at that height, the right one is lowest. +//! +//! Measured on a synthetic 720p -> 1080p Catmull-Rom master: 2.8e-8 at 720 +//! against ~2e-4 elsewhere, and Catmull-Rom 2.8e-8 against 1.9e-5 for the next +//! kernel. A real Blu-ray's dip is shallower (grain, compression, credits), so +//! a height is reported as found only when its error is well below its +//! neighbours'; a source that was never upscaled has no dip at all, and is +//! reported as such rather than given a guess that would damage it. + +use std::fs; +use std::process::{Command, Stdio}; + +use anyhow::{bail, Context, Result}; +use serde::{Deserialize, Serialize}; + +use crate::dependency_locator::DependencyLocator; +use crate::models::VideoJob; +use crate::source_decode; + +/// Marker for the analysis script's one result line on stderr. +const RESULT_MARKER: &str = "NATIVE_RES_RESULT:"; + +/// What the detection found. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct NativeResReport { + /// Whether a native resolution stands out clearly enough to use. + pub found: bool, + pub height: i32, + pub width: i32, + /// The Descale pass's kernel name (`bilinear`, `bicubic`, `lanczos`, + /// `spline16`, `spline36`, `spline64`) and its parameters. + pub kernel: String, + pub bicubic_b: f64, + pub bicubic_c: f64, + pub lanczos_taps: i32, + /// The best height's error over the median of its neighbours': near 0 is + /// a sharp dip, near 1 none at all. + pub dip: f64, + /// Every kernel tried at that height, best first: (label, mean error). + pub kernels: Vec<(String, f64)>, + /// The height scan, for display: (height, error). + pub curve: Vec<(i32, f64)>, + /// The source frames analysed. + pub frames: Vec, +} + +/// A dip at or below this fraction of its neighbours' error counts as found. +pub const DIP_THRESHOLD: f64 = 0.5; + +/// Which source frames to analyse: a few spread through the clip (not the very +/// start or end, where titles, fades and black frames are likeliest). +pub fn sample_frames(total: Option, start: Option, end: Option) -> Vec { + let first = start.unwrap_or(0); + let last = match (end, total) { + (Some(e), _) => e, + (None, Some(t)) if t > 0 => t - 1, + _ => return vec![first as i32], + }; + if last <= first { + return vec![first as i32]; + } + let span = last - first; + [0.2, 0.4, 0.6, 0.8] + .iter() + .map(|f| (first + (span as f64 * f).round() as u64) as i32) + .collect() +} + +/// The analysis script. `{paths}` are raw GRAY16 frames of `{w}`x`{h}`. +fn analysis_script(paths: &[String], width: i32, height: i32) -> String { + let paths = paths.iter().map(|p| crate::gpu_plugin::py_str(std::path::Path::new(p))).collect::>().join(", "); + format!( + r#"import json, sys, ctypes +import vapoursynth as vs +core = vs.core +W, H = {width}, {height} +PATHS = [{paths}] + +def load(path): + data = open(path, 'rb').read() + blank = core.std.BlankClip(width=W, height=H, format=vs.GRAY16, length=1) + def fill(n, f): + fo = f.copy() + ptr, stride = fo.get_write_ptr(0).value, fo.get_stride(0) + for y in range(H): + ctypes.memmove(ptr + y * stride, data[y * W * 2:(y + 1) * W * 2], W * 2) + return fo + return core.resize.Point(core.std.ModifyFrame(blank, blank, fill), format=vs.GRAYS) + +frames = [load(p) for p in PATHS] + +# (label, descale kernel, kernel name and parameters for the Descale pass) +KERNELS = [ + ('Bilinear', lambda c, w, h: core.descale.Debilinear(c, w, h), lambda c, w, h: core.resize.Bilinear(c, w, h), ('bilinear', 0, 0, 3)), + ('Bicubic (Catmull-Rom)', lambda c, w, h: core.descale.Debicubic(c, w, h, b=0, c=0.5), lambda c, w, h: core.resize.Bicubic(c, w, h, filter_param_a=0, filter_param_b=0.5), ('bicubic', 0, 0.5, 3)), + ('Bicubic (Mitchell)', lambda c, w, h: core.descale.Debicubic(c, w, h, b=1/3, c=1/3), lambda c, w, h: core.resize.Bicubic(c, w, h, filter_param_a=1/3, filter_param_b=1/3), ('bicubic', 1/3, 1/3, 3)), + ('Bicubic (Hermite)', lambda c, w, h: core.descale.Debicubic(c, w, h, b=0, c=0), lambda c, w, h: core.resize.Bicubic(c, w, h, filter_param_a=0, filter_param_b=0), ('bicubic', 0, 0, 3)), + ('Bicubic (sharp, b=0 c=1)', lambda c, w, h: core.descale.Debicubic(c, w, h, b=0, c=1), lambda c, w, h: core.resize.Bicubic(c, w, h, filter_param_a=0, filter_param_b=1), ('bicubic', 0, 1, 3)), + ('Bicubic (b=-0.5 c=0.25)', lambda c, w, h: core.descale.Debicubic(c, w, h, b=-0.5, c=0.25), lambda c, w, h: core.resize.Bicubic(c, w, h, filter_param_a=-0.5, filter_param_b=0.25), ('bicubic', -0.5, 0.25, 3)), + ('Lanczos (2 taps)', lambda c, w, h: core.descale.Delanczos(c, w, h, taps=2), lambda c, w, h: core.resize.Lanczos(c, w, h, filter_param_a=2), ('lanczos', 0, 0.5, 2)), + ('Lanczos (3 taps)', lambda c, w, h: core.descale.Delanczos(c, w, h, taps=3), lambda c, w, h: core.resize.Lanczos(c, w, h, filter_param_a=3), ('lanczos', 0, 0.5, 3)), + ('Lanczos (4 taps)', lambda c, w, h: core.descale.Delanczos(c, w, h, taps=4), lambda c, w, h: core.resize.Lanczos(c, w, h, filter_param_a=4), ('lanczos', 0, 0.5, 4)), + ('Spline16', lambda c, w, h: core.descale.Despline16(c, w, h), lambda c, w, h: core.resize.Spline16(c, w, h), ('spline16', 0, 0.5, 3)), + ('Spline36', lambda c, w, h: core.descale.Despline36(c, w, h), lambda c, w, h: core.resize.Spline36(c, w, h), ('spline36', 0, 0.5, 3)), + ('Spline64', lambda c, w, h: core.descale.Despline64(c, w, h), lambda c, w, h: core.resize.Spline64(c, w, h), ('spline64', 0, 0.5, 3)), +] + +def width_for(h): + return int(round(h * W / H)) + +def error_clip(src, kernel, h): + # Each pixel's error is capped (2 levels in 8-bit terms) before averaging, + # so detail that fits at no height — credits, text, overlays drawn at full + # resolution — adds the same small amount everywhere instead of swamping + # the dip, while the misfit of a wrong height still counts in full across + # the rest of the picture. + _, de, fwd, _ = kernel + diff = core.std.Expr([fwd(de(src, width_for(h), h), W, H), src], 'x y - abs 0.0078 min') + return core.std.PlaneStats(diff) + +def errors(clips): + # One spliced clip, read with frames(): VapourSynth evaluates the + # candidates in parallel rather than one get_frame at a time. + joined = core.std.Splice(clips) if len(clips) > 1 else clips[0] + return [f.props['PlaneStatsAverage'] for f in joined.frames()] + +# 1. Scan every height from half the source's up, on the first frame, with one +# kernel from each family studios use (a master only dips under a kernel +# close to its own); keep the best error per height. +heights = list(range(max(H // 2, 16), H)) +probe = frames[0] +scan = {{}} +for kernel in (KERNELS[0], KERNELS[1], KERNELS[7], KERNELS[10]): + for h, e in zip(heights, errors([error_clip(probe, kernel, h) for h in heights])): + scan[h] = min(e, scan.get(h, e)) + +# 2. The height that dips furthest below its neighbours (+-15, skipping +-2). +# Only heights with neighbours on BOTH sides are candidates: the error falls +# steadily toward the source height (H-1 is nearly an identity resample), so +# a one-sided window would read that slope as a dip. +def dip(h): + near = sorted(scan[x] for x in range(h - 15, h + 16) if x in scan and abs(x - h) > 2) + if not near: + return 1.0 + med = near[len(near) // 2] + return scan[h] / med if med > 0 else 1.0 +candidates = [h for h in heights if (h - 15) in scan and (h + 15) in scan] +best_h = min(candidates, key=dip) + +# 3. Every kernel at that height, averaged over all the sampled frames. +ranked = [] +for kernel in KERNELS: + errs = errors([error_clip(f, kernel, best_h) for f in frames]) + ranked.append((kernel, sum(errs) / len(errs))) +ranked.sort(key=lambda kv: kv[1]) +(label, _, _, (name, b, c, taps)), _ = ranked[0] + +result = dict( + height=best_h, width=width_for(best_h), kernel=name, bicubicB=b, bicubicC=c, lanczosTaps=taps, + dip=dip(best_h), kernels=[[k[0], e] for k, e in ranked], + curve=[[h, scan[h]] for h in heights], +) +print('{marker}' + json.dumps(result), file=sys.stderr, flush=True) +core.std.BlankClip(width=16, height=16, length=1).set_output() +"#, + marker = RESULT_MARKER, + ) +} + +/// Decode the sample frames' luma and run the analysis. +pub fn detect(deps: &DependencyLocator, job: &VideoJob) -> Result { + let ffmpeg = deps.ffmpeg_path()?; + let vspipe = deps.vspipe_path()?; + let env = deps.build_environment(); + let (width, height) = match (job.input_width, job.input_height) { + (Some(w), Some(h)) if w > 0 && h > 0 => (w, h), + _ => bail!("The video's size is unknown; re-add it to the queue."), + }; + if height < 64 { + bail!("The video is too small to analyse ({width}x{height})."); + } + let fps = job.input_frame_rate.unwrap_or(29.97); + let frames = sample_frames( + job.total_frames.map(|f| f.max(0) as u64), + job.start_frame.map(|f| f.max(0) as u64), + job.end_frame.map(|f| f.max(0) as u64), + ); + + let dir = std::env::temp_dir().join(format!("vapourbox_native_{}", job.id)); + fs::create_dir_all(&dir).with_context(|| format!("creating {dir:?}"))?; + let result = (|| -> Result { + let mut paths = Vec::new(); + for &frame in &frames { + let path = dir.join(format!("frame_{frame}.gray16")); + let seek = ((frame as f64 - 0.5) / fps).max(0.0); + let out = Command::new(&ffmpeg) + .args(source_decode::preview_decoder_args( + &job.input_path, + seek, + 1, + width, + height, + "gray16le", + path.to_string_lossy().as_ref(), + )) + .envs(&env) + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .output() + .context("running ffmpeg")?; + if !out.status.success() || !path.exists() { + let stderr = String::from_utf8_lossy(&out.stderr); + if let Some(msg) = source_decode::explain_decoder_failure(&stderr, width, height) { + bail!("{msg}"); + } + bail!("Could not decode frame {frame}: {stderr}"); + } + paths.push(path.to_string_lossy().to_string()); + } + + let script = dir.join("detect.vpy"); + fs::write(&script, analysis_script(&paths, width, height))?; + let out = Command::new(&vspipe) + .args(["--info", script.to_string_lossy().as_ref(), "-"]) + .envs(&env) + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .output() + .context("running vspipe")?; + let stderr = String::from_utf8_lossy(&out.stderr); + let line = stderr + .lines() + .find_map(|l| l.trim().strip_prefix(RESULT_MARKER)) + .with_context(|| format!("the analysis did not finish: {}", tail(&stderr)))?; + let raw: serde_json::Value = serde_json::from_str(line)?; + Ok(report_from(&raw, frames.clone())) + })(); + let _ = fs::remove_dir_all(&dir); + result +} + +fn tail(s: &str) -> String { + s.lines().rev().take(4).collect::>().into_iter().rev().collect::>().join(" | ") +} + +/// The script's JSON as a report, with the found/not-found decision made here. +pub fn report_from(raw: &serde_json::Value, frames: Vec) -> NativeResReport { + let f = |k: &str| raw.get(k).and_then(|v| v.as_f64()).unwrap_or(0.0); + let pairs = |k: &str| -> Vec<(serde_json::Value, f64)> { + raw.get(k) + .and_then(|v| v.as_array()) + .map(|a| { + a.iter() + .filter_map(|p| Some((p.get(0)?.clone(), p.get(1)?.as_f64()?))) + .collect() + }) + .unwrap_or_default() + }; + let dip = raw.get("dip").and_then(|v| v.as_f64()).unwrap_or(1.0); + NativeResReport { + found: dip <= DIP_THRESHOLD, + height: f("height") as i32, + width: f("width") as i32, + kernel: raw.get("kernel").and_then(|v| v.as_str()).unwrap_or("bicubic").to_string(), + bicubic_b: f("bicubicB"), + bicubic_c: f("bicubicC"), + lanczos_taps: f("lanczosTaps") as i32, + dip, + kernels: pairs("kernels") + .into_iter() + .map(|(k, e)| (k.as_str().unwrap_or_default().to_string(), e)) + .collect(), + curve: pairs("curve") + .into_iter() + .map(|(h, e)| (h.as_i64().unwrap_or_default() as i32, e)) + .collect(), + frames, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn samples_spread_through_the_clip_or_its_trim() { + assert_eq!(sample_frames(Some(1001), None, None), vec![200, 400, 600, 800]); + assert_eq!(sample_frames(Some(1000), Some(100), Some(199)), vec![120, 140, 159, 179]); + assert_eq!(sample_frames(None, None, None), vec![0]); + assert_eq!(sample_frames(Some(1), None, None), vec![0]); + } + + #[test] + fn found_only_for_a_clear_dip() { + let raw = |dip: f64| serde_json::json!({ + "height": 720, "width": 1280, "kernel": "bicubic", "bicubicB": 0.0, "bicubicC": 0.5, + "lanczosTaps": 3, "dip": dip, + "kernels": [["Bicubic (Catmull-Rom)", 1e-8], ["Bilinear", 6e-5]], + "curve": [[719, 2e-4], [720, 1e-8]], + }); + let r = report_from(&raw(0.0001), vec![5]); + assert!(r.found); + assert_eq!((r.height, r.width, r.kernel.as_str()), (720, 1280, "bicubic")); + assert_eq!(r.kernels[0].0, "Bicubic (Catmull-Rom)"); + assert_eq!(r.curve[1], (720, 1e-8)); + assert!(!report_from(&raw(0.93), vec![5]).found); + } + + #[test] + fn the_script_scans_from_half_height_and_names_every_pass_kernel() { + let s = analysis_script(&["/a/f.gray16".into()], 1920, 1080); + assert!(s.contains("W, H = 1920, 1080")); + assert!(s.contains("PATHS = ['/a/f.gray16']")); + assert!(s.contains("range(max(H // 2, 16), H)")); + for k in ["'bilinear'", "'bicubic'", "'lanczos'", "'spline16'", "'spline36'", "'spline64'"] { + assert!(s.contains(k), "{k}"); + } + assert!(s.contains(RESULT_MARKER)); + } +} diff --git a/worker/src/pipeline_executor.rs b/worker/src/pipeline_executor.rs index f7939c57..97ed5bbb 100644 --- a/worker/src/pipeline_executor.rs +++ b/worker/src/pipeline_executor.rs @@ -148,6 +148,7 @@ fn preview_window(target: i32, radius: i32) -> (i32, i32, i32) { (first, local, num_frames) } +use crate::gpu_plugin::GpuPlugin; use crate::dependency_locator::DependencyLocator; use crate::models::{AspectDeclaration, AudioMode, ContainerFormat, DeinterlaceMethod, EncoderFamily, LogLevel, ProgressInfo, SubtitleOutput, VideoCodec, VideoJob}; use crate::pixel_format; @@ -1250,9 +1251,19 @@ impl PipelineExecutor { .with_opencl_available(self.deps.opencl_available()) .with_knlm_available(self.deps.knlm_available()) .with_rife(if job.effective_pipeline().uses_rife() { - self.deps.rife_runtime() + self.deps.gpu_plugin_runtime(GpuPlugin::Rife) } else { Err("RIFE is not used by this job.".into()) + }) + .with_upscale(if job.effective_pipeline().uses_ai_upscale() { + self.deps.gpu_plugin_runtime(GpuPlugin::Upscale) + } else { + Err("AI upscaling is not used by this job.".into()) + }) + .with_knlm(if job.effective_pipeline().uses_knlm() { + self.deps.knlm_runtime() + } else { + Err("KNLMeans is not used by this job.".into()) }); let (fps_num, fps_den) = script_generator.frame_rate_to_rational(frame_rate); diff --git a/worker/src/rife.rs b/worker/src/rife.rs deleted file mode 100644 index 3ba396e8..00000000 --- a/worker/src/rife.rs +++ /dev/null @@ -1,582 +0,0 @@ -//! RIFE frame interpolation: where its files live, and whether it can run here. -//! -//! RIFE (`rife.RIFE`, ncnn on Vulkan) is the one Frame Rate method that needs -//! more than the deps bundle: a GPU with a working Vulkan driver, and on Windows -//! the MSVC OpenMP runtime. Without a Vulkan implementation the plugin does not -//! raise — it segfaults inside `ncnn::create_gpu_instance` — so a job cannot be -//! allowed to find out by trying. Instead this module runs the plugin once in a -//! throwaway vspipe, records how far it got, and turns the outcome into a reason -//! with a remedy the user can act on. The app shows that message the moment -//! RIFE is selected (`--probe-rife`), and the worker refuses a RIFE job with the -//! same message rather than emitting a script that would crash. -//! -//! The plugin lives in `vapoursynth/rife/`, outside the autoload directory, and -//! the script loads it explicitly — so nothing about it (a missing runtime DLL, -//! a driver that faults on load, AVX in a static initializer) can reach a job -//! that did not ask for RIFE. - -use serde::{Deserialize, Serialize}; -use std::collections::HashMap; -use std::io::Read; -use std::path::{Path, PathBuf}; -use std::process::{Command, Stdio}; -use std::time::{Duration, Instant}; - -/// The bundled model: RIFE 4.26, ncnn conversion, ensemble off. -pub const MODEL_DIR_NAME: &str = "rife-v4.26_ensembleFalse"; - -/// Marker the probe script prints to stderr as it passes each stage, so a -/// process that died without a message still says where it died. -const STAGE_MARKER: &str = "RIFE_PROBE:"; - -/// How long the probe may take. GPU initialisation and the first shader -/// compile take a second or two; a driver that hangs must not hang the app. -const PROBE_TIMEOUT: Duration = Duration::from_secs(60); - -/// Whether the deps bundle for this platform ships RIFE at all. -/// -/// macOS x64 is excluded because the only prebuilt dylib targets macOS 13 -/// (the bundle's floor is 12) and runs AVX in a static initializer; linux-arm64 -/// because upstream publishes no build. Both need a source build first. -pub fn platform_ships_rife(platform_suffix: &str) -> bool { - matches!(platform_suffix, "macos-arm64" | "windows-x64" | "linux-x64") -} - -/// The files RIFE needs, as laid out under `deps//vapoursynth/rife/`. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct RifeFiles { - pub plugin: PathBuf, - pub model_dir: PathBuf, - /// A library to load into the process before the plugin: MoltenVK on - /// macOS, where the plugin resolves Vulkan from whatever is already loaded. - pub preload: Option, -} - -impl RifeFiles { - pub fn in_vapoursynth_dir(vs_dir: &Path, platform_suffix: &str) -> Self { - let dir = vs_dir.join("rife"); - let (plugin, preload) = if platform_suffix.starts_with("macos") { - ("librife.dylib", Some(dir.join("libMoltenVK.dylib"))) - } else if platform_suffix.starts_with("windows") { - ("librife.dll", None) - } else { - ("librife.so", None) - }; - Self { - plugin: dir.join(plugin), - model_dir: dir.join("models").join(MODEL_DIR_NAME), - preload, - } - } - - /// Every file that must exist, in the order a user would want to hear - /// about them. - pub fn required(&self) -> Vec { - let mut files = vec![self.plugin.clone()]; - files.extend(self.preload.iter().cloned()); - files.push(self.model_dir.join("flownet.param")); - files.push(self.model_dir.join("flownet.bin")); - files - } - - pub fn missing(&self) -> Vec { - self.required().into_iter().filter(|p| !p.is_file()).collect() - } -} - -/// Why RIFE cannot run here. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub enum RifeReason { - /// No build of RIFE ships for this platform. - UnsupportedPlatform, - /// The installed deps bundle predates RIFE, or is missing its files. - NotInstalled, - /// No usable Vulkan device or driver. - NoVulkan, - /// A system library the plugin links against is not installed. - MissingLibrary, - /// The plugin would not load, for a reason the probe could not narrow. - LoadFailed, - /// The probe process crashed after the GPU was initialised. - Crashed, - /// The plugin used an instruction this CPU does not have (SIGILL, or - /// Windows' 0xC000001D) — an x86 machine older than its build target. - UnsupportedCpu, - /// `VAPOURBOX_DISABLE_VULKAN` is set. - Disabled, - /// The probe itself could not be run or did not finish. - ProbeFailed, -} - -/// Whether RIFE runs here, and if not, why. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct RifeStatus { - pub available: bool, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub reason: Option, - /// The specific thing that failed (a missing file, a library name, the - /// error text), shown under the remedy. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub detail: Option, -} - -impl RifeStatus { - pub fn available() -> Self { - Self { available: true, reason: None, detail: None } - } - - pub fn unavailable(reason: RifeReason, detail: impl Into>) -> Self { - Self { available: false, reason: Some(reason), detail: detail.into() } - } - - /// What to tell the user: what is wrong and what to do about it. `None` - /// when RIFE is available. The app shows this verbatim and the worker - /// fails a RIFE job with it, so there is one wording, written here. - pub fn message(&self, platform_suffix: &str) -> Option { - let reason = self.reason?; - let os = platform_suffix.split('-').next().unwrap_or(""); - let text = match reason { - RifeReason::UnsupportedPlatform => { - let which = match platform_suffix { - "macos-x64" => "Intel Macs", - "linux-arm64" => "Linux on ARM", - "windows-arm64" => "Windows on ARM", - _ => "this platform", - }; - format!( - "RIFE isn't available for {which} yet: no build of it ships with \ - VapourBox for this platform. Use Motion interpolation instead." - ) - } - RifeReason::NotInstalled => "RIFE's files aren't in the installed dependency \ - bundle, which is probably older than this version of VapourBox. Restart \ - VapourBox to let it download the current dependencies." - .to_string(), - RifeReason::NoVulkan => match os { - "windows" => "RIFE needs a graphics card with Vulkan support, and none was \ - found. Install or update the driver from your GPU's maker (NVIDIA, AMD \ - or Intel) — Vulkan support comes with it — then restart VapourBox. \ - Remote desktop sessions and virtual machines usually have no Vulkan \ - device." - .to_string(), - "linux" => "RIFE needs a graphics card with a Vulkan driver, and none was \ - found. Install your distribution's Vulkan driver and loader (for \ - example mesa-vulkan-drivers and libvulkan1 on Debian/Ubuntu, or \ - mesa-vulkan-drivers and vulkan-loader on Fedora; NVIDIA's own driver \ - includes Vulkan), then restart VapourBox." - .to_string(), - _ => "RIFE could not open the GPU through Metal. This happens in virtual \ - machines and some remote sessions; on a normal login it should work, \ - so try restarting VapourBox." - .to_string(), - }, - RifeReason::MissingLibrary => match os { - "windows" => "RIFE needs the Microsoft Visual C++ Redistributable (x64), \ - which isn't installed. Install the latest version from Microsoft \ - (search for \"Visual C++ Redistributable latest supported downloads\"), \ - then restart VapourBox." - .to_string(), - _ => "RIFE needs a system library that isn't installed. Install it with \ - your package manager (libgomp is in the libgomp1 or libgomp package), \ - then restart VapourBox." - .to_string(), - }, - RifeReason::LoadFailed => "RIFE's plugin failed to load on this computer. \ - Restart VapourBox to re-check its dependencies; if this persists, please \ - report it with the details below." - .to_string(), - RifeReason::Crashed => "RIFE crashed while testing the GPU. Updating your \ - graphics driver usually fixes this; restart VapourBox afterwards." - .to_string(), - RifeReason::UnsupportedCpu => "RIFE's plugin needs a newer processor than \ - this computer has (it uses instructions this CPU lacks). Use Motion \ - interpolation instead." - .to_string(), - RifeReason::Disabled => "RIFE is turned off by the VAPOURBOX_DISABLE_VULKAN \ - environment variable." - .to_string(), - RifeReason::ProbeFailed => "VapourBox couldn't check whether RIFE runs on this \ - computer. Restart VapourBox to try again." - .to_string(), - }; - Some(match &self.detail { - Some(d) if !d.is_empty() => format!("{text}\n\nDetails: {d}"), - _ => text, - }) - } - - /// The probe result as the JSON `--probe-rife` prints, message included. - pub fn to_json(&self, platform_suffix: &str) -> serde_json::Value { - let mut v = serde_json::to_value(self).unwrap_or_default(); - if let (Some(msg), Some(obj)) = (self.message(platform_suffix), v.as_object_mut()) { - obj.insert("message".into(), serde_json::Value::String(msg)); - } - v - } -} - -/// Everything the platform and file checks can decide without running -/// anything. `None` means "it should work — run the probe". -pub fn precheck(platform_suffix: &str, files: &RifeFiles) -> Option { - if !platform_ships_rife(platform_suffix) { - return Some(RifeStatus::unavailable(RifeReason::UnsupportedPlatform, None)); - } - let missing = files.missing(); - if !missing.is_empty() { - let list = missing.iter().map(|p| p.display().to_string()).collect::>(); - return Some(RifeStatus::unavailable( - RifeReason::NotInstalled, - format!("missing {}", list.join(", ")), - )); - } - None -} - -/// A Python string literal for a path, for substitution into a script. -pub fn py_str(path: &Path) -> String { - let s = path.to_string_lossy(); - let mut out = String::with_capacity(s.len() + 2); - out.push('\''); - for c in s.chars() { - match c { - '\\' => out.push_str("\\\\"), - '\'' => out.push_str("\\'"), - '\n' => out.push_str("\\n"), - '\r' => out.push_str("\\r"), - c => out.push(c), - } - } - out.push('\''); - out -} - -/// The probe: load the plugin, create the GPU instance, and render one -/// interpolated frame — the three places it has been seen to fail. -pub fn probe_script(files: &RifeFiles, platform_suffix: &str) -> String { - // Shared libraries whose absence makes the plugin fail to load with only a - // generic "module not found". Checked first so the user is told which one. - let runtime_libs: &[&str] = if platform_suffix.starts_with("windows") { - &["vcomp140.dll", "msvcp140.dll", "msvcp140_atomic_wait.dll", "vcruntime140.dll", "vcruntime140_1.dll"] - } else if platform_suffix.starts_with("linux") { - &["libgomp.so.1"] - } else { - &[] - }; - let libs = runtime_libs.iter().map(|l| format!("'{l}'")).collect::>().join(", "); - let preload = files.preload.as_deref().map(py_str).unwrap_or_else(|| "None".into()); - format!( - "import sys, ctypes\n\ - def stage(s):\n print('{marker}' + s, file=sys.stderr, flush=True)\n\ - for lib in [{libs}]:\n try:\n ctypes.CDLL(lib)\n except OSError:\n stage('missing:' + lib)\n\ - stage('preload')\n\ - if {preload}:\n ctypes.CDLL({preload}, mode=ctypes.RTLD_GLOBAL)\n\ - import vapoursynth as vs\n\ - core = vs.core\n\ - stage('load')\n\ - if not hasattr(core, 'rife'):\n core.std.LoadPlugin({plugin})\n\ - stage('init')\n\ - clip = core.std.BlankClip(width=64, height=64, format=vs.RGBS, length=2, fpsnum=25, fpsden=1)\n\ - clip = core.rife.RIFE(clip, model_path={model}, fps_num=50, fps_den=1)\n\ - stage('render')\n\ - clip.set_output()\n", - marker = STAGE_MARKER, - plugin = py_str(&files.plugin), - model = py_str(&files.model_dir), - ) -} - -/// How the probe process ended. -#[derive(Debug, Clone, Default, PartialEq, Eq)] -pub struct ProbeExit { - pub success: bool, - pub timed_out: bool, - /// Human-readable exit status for a failure (`signal 11 (SIGSEGV)` …). - pub status: String, -} - -/// Turn a finished probe into a status. Pure, so every branch is testable. -pub fn classify(exit: &ProbeExit, stderr: &str) -> RifeStatus { - if exit.success { - return RifeStatus::available(); - } - if exit.timed_out { - return RifeStatus::unavailable( - RifeReason::ProbeFailed, - format!("the GPU test did not finish within {}s", PROBE_TIMEOUT.as_secs()), - ); - } - - let stages: Vec<&str> = stderr - .lines() - .filter_map(|l| l.trim().strip_prefix(STAGE_MARKER)) - .collect(); - let missing: Vec<&str> = stages.iter().filter_map(|s| s.strip_prefix("missing:")).collect(); - let last = stages.iter().rev().find(|s| !s.starts_with("missing:")).copied(); - let error = last_error_line(stderr); - - // An illegal instruction is the CPU, wherever it happened. - if exit.status.starts_with("signal 4 ") || exit.status.contains("illegal instruction") { - return RifeStatus::unavailable(RifeReason::UnsupportedCpu, exit.status.clone()); - } - - if stderr.contains("failed to create GPU instance") || stderr.contains("invalid GPU device") { - return RifeStatus::unavailable(RifeReason::NoVulkan, error); - } - if stderr.contains("failed to load model") { - return RifeStatus::unavailable(RifeReason::NotInstalled, error); - } - match last { - // The plugin would not load. A runtime library the pre-check could not - // open is the likeliest cause, and the one the user can fix. - Some("load") if !missing.is_empty() => RifeStatus::unavailable( - RifeReason::MissingLibrary, - format!("could not load {}", missing.join(", ")), - ), - Some("load") | Some("preload") => RifeStatus::unavailable( - RifeReason::LoadFailed, - error.or_else(|| Some(format!("vspipe exited with {}", exit.status))), - ), - // Died creating the GPU instance without an error: no Vulkan loader or - // driver for ncnn to call into. - Some("init") => RifeStatus::unavailable( - RifeReason::NoVulkan, - error.or_else(|| Some(format!("the GPU could not be initialised ({})", exit.status))), - ), - Some("render") => RifeStatus::unavailable( - RifeReason::Crashed, - error.or_else(|| Some(format!("vspipe exited with {}", exit.status))), - ), - _ => RifeStatus::unavailable( - RifeReason::ProbeFailed, - error.or_else(|| Some(format!("vspipe exited with {}", exit.status))), - ), - } -} - -/// The last line of a Python/VapourSynth error, which names the actual failure. -fn last_error_line(stderr: &str) -> Option { - stderr - .lines() - .map(str::trim) - .filter(|l| !l.is_empty() && !l.starts_with(STAGE_MARKER) && !l.starts_with('[')) - .filter(|l| l.contains("Error") || l.contains("error") || l.contains("Failed")) - .last() - .map(|l| l.chars().take(400).collect()) -} - -/// Run the probe script through vspipe and classify the result. -pub fn run_probe( - vspipe: &Path, - env: &HashMap, - files: &RifeFiles, - platform_suffix: &str, -) -> RifeStatus { - let script = probe_script(files, platform_suffix); - let tmp = std::env::temp_dir().join(format!("vapourbox_rife_probe_{}.vpy", std::process::id())); - if let Err(e) = std::fs::write(&tmp, script) { - return RifeStatus::unavailable(RifeReason::ProbeFailed, format!("could not write the probe script: {e}")); - } - let exit_and_stderr = run_with_timeout(vspipe, &tmp, env); - let _ = std::fs::remove_file(&tmp); - match exit_and_stderr { - Ok((exit, stderr)) => classify(&exit, &stderr), - Err(e) => RifeStatus::unavailable(RifeReason::ProbeFailed, format!("could not run vspipe: {e}")), - } -} - -fn run_with_timeout( - vspipe: &Path, - script: &Path, - env: &HashMap, -) -> std::io::Result<(ProbeExit, String)> { - // Frame 1 of 2x output is the first interpolated one; frame 0 is a copy - // and would never touch the GPU. - let mut child = Command::new(vspipe) - .args(["-s", "1", "-e", "1"]) - .arg(script) - .arg("-") - .envs(env) - .env("MVK_CONFIG_LOG_LEVEL", "0") - .stdout(Stdio::null()) - .stderr(Stdio::piped()) - .spawn()?; - let mut stderr = child.stderr.take(); - let reader = std::thread::spawn(move || { - let mut s = String::new(); - if let Some(e) = stderr.as_mut() { - let _ = e.read_to_string(&mut s); - } - s - }); - let start = Instant::now(); - let status = loop { - if let Some(status) = child.try_wait()? { - break Some(status); - } - if start.elapsed() > PROBE_TIMEOUT { - let _ = child.kill(); - let _ = child.wait(); - break None; - } - std::thread::sleep(Duration::from_millis(50)); - }; - let stderr = reader.join().unwrap_or_default(); - let exit = match status { - Some(s) => ProbeExit { - success: s.success(), - timed_out: false, - status: crate::pipeline_executor::format_exit_status(&s), - }, - None => ProbeExit { success: false, timed_out: true, status: "timed out".into() }, - }; - Ok((exit, stderr)) -} - -#[cfg(test)] -mod tests { - use super::*; - - fn failed(status: &str) -> ProbeExit { - ProbeExit { success: false, timed_out: false, status: status.into() } - } - - #[test] - fn layout_per_platform() { - let mac = RifeFiles::in_vapoursynth_dir(Path::new("/d/vapoursynth"), "macos-arm64"); - assert_eq!(mac.plugin, Path::new("/d/vapoursynth/rife/librife.dylib")); - assert_eq!(mac.preload.as_deref(), Some(Path::new("/d/vapoursynth/rife/libMoltenVK.dylib"))); - assert!(mac.model_dir.ends_with("rife/models/rife-v4.26_ensembleFalse")); - let win = RifeFiles::in_vapoursynth_dir(Path::new("D:/v"), "windows-x64"); - assert!(win.plugin.ends_with("rife/librife.dll")); - assert_eq!(win.preload, None); - let linux = RifeFiles::in_vapoursynth_dir(Path::new("/v"), "linux-x64"); - assert!(linux.plugin.ends_with("rife/librife.so")); - // MoltenVK is required on macOS, and only there. - assert_eq!(mac.required().len(), 4); - assert_eq!(linux.required().len(), 3); - } - - #[test] - fn unsupported_platforms_are_reported_before_anything_runs() { - let files = RifeFiles::in_vapoursynth_dir(Path::new("/nonexistent"), "macos-x64"); - for p in ["macos-x64", "linux-arm64", "windows-arm64"] { - let s = precheck(p, &files).expect("must decide without probing"); - assert_eq!(s.reason, Some(RifeReason::UnsupportedPlatform), "{p}"); - } - // A shipping platform with no files is "not installed", not unsupported. - let files = RifeFiles::in_vapoursynth_dir(Path::new("/nonexistent"), "linux-x64"); - let s = precheck("linux-x64", &files).unwrap(); - assert_eq!(s.reason, Some(RifeReason::NotInstalled)); - assert!(s.detail.unwrap().contains("librife.so")); - } - - #[test] - fn every_reason_has_a_message_on_every_platform() { - let reasons = [ - RifeReason::UnsupportedPlatform, - RifeReason::NotInstalled, - RifeReason::NoVulkan, - RifeReason::MissingLibrary, - RifeReason::LoadFailed, - RifeReason::Crashed, - RifeReason::UnsupportedCpu, - RifeReason::Disabled, - RifeReason::ProbeFailed, - ]; - for platform in ["macos-arm64", "macos-x64", "windows-x64", "linux-x64", "linux-arm64"] { - for r in reasons { - let m = RifeStatus::unavailable(r, None).message(platform).unwrap(); - assert!(m.len() > 40, "{platform} {r:?}: {m}"); - } - } - assert_eq!(RifeStatus::available().message("linux-x64"), None); - let with_detail = RifeStatus::unavailable(RifeReason::MissingLibrary, "could not load vcomp140.dll".to_string()); - let m = with_detail.message("windows-x64").unwrap(); - assert!(m.contains("Visual C++ Redistributable") && m.contains("vcomp140.dll")); - } - - #[test] - fn json_carries_the_message() { - let v = RifeStatus::unavailable(RifeReason::NoVulkan, None).to_json("windows-x64"); - assert_eq!(v["available"], false); - assert_eq!(v["reason"], "noVulkan"); - assert!(v["message"].as_str().unwrap().contains("Vulkan")); - let ok = RifeStatus::available().to_json("windows-x64"); - assert_eq!(ok, serde_json::json!({ "available": true })); - } - - #[test] - fn classify_success() { - let ok = ProbeExit { success: true, ..Default::default() }; - assert!(classify(&ok, "RIFE_PROBE:render\n").available); - } - - #[test] - fn classify_segfault_creating_the_gpu_instance_is_no_vulkan() { - // What macOS does with no MoltenVK in the process, reproduced 2026-10-24. - let s = classify(&failed("signal 11 (SIGSEGV)"), "RIFE_PROBE:preload\nRIFE_PROBE:load\nRIFE_PROBE:init\n"); - assert_eq!(s.reason, Some(RifeReason::NoVulkan)); - } - - #[test] - fn classify_plugin_errors() { - let gpu = "RIFE_PROBE:init\nvapoursynth.Error: RIFE: failed to create GPU instance\n"; - assert_eq!(classify(&failed("exit code 1"), gpu).reason, Some(RifeReason::NoVulkan)); - let dev = "RIFE_PROBE:init\nvapoursynth.Error: RIFE: invalid GPU device\n"; - assert_eq!(classify(&failed("exit code 1"), dev).reason, Some(RifeReason::NoVulkan)); - let model = "RIFE_PROBE:init\nvapoursynth.Error: RIFE: failed to load model\n"; - assert_eq!(classify(&failed("exit code 1"), model).reason, Some(RifeReason::NotInstalled)); - } - - #[test] - fn classify_load_failure_names_the_missing_runtime() { - let err = "RIFE_PROBE:missing:vcomp140.dll\nRIFE_PROBE:preload\nRIFE_PROBE:load\n\ - vapoursynth.Error: Failed to load C:\\deps\\librife.dll. GetLastError() returned 126.\n"; - let s = classify(&failed("exit code 1"), err); - assert_eq!(s.reason, Some(RifeReason::MissingLibrary)); - assert!(s.detail.unwrap().contains("vcomp140.dll")); - // With every runtime present, the loader's own error is the detail. - let s = classify(&failed("exit code 1"), &err.replace("RIFE_PROBE:missing:vcomp140.dll\n", "")); - assert_eq!(s.reason, Some(RifeReason::LoadFailed)); - assert!(s.detail.unwrap().contains("GetLastError() returned 126")); - } - - #[test] - fn classify_crash_while_rendering_and_timeouts() { - let s = classify(&failed("signal 11 (SIGSEGV)"), "RIFE_PROBE:init\nRIFE_PROBE:render\n"); - assert_eq!(s.reason, Some(RifeReason::Crashed)); - assert!(s.detail.unwrap().contains("SIGSEGV")); - let t = ProbeExit { success: false, timed_out: true, status: "timed out".into() }; - assert_eq!(classify(&t, "").reason, Some(RifeReason::ProbeFailed)); - // vspipe that never reached the script at all. - assert_eq!(classify(&failed("exit code 1"), "").reason, Some(RifeReason::ProbeFailed)); - } - - #[test] - fn classify_illegal_instruction_is_the_cpu() { - let sigill = classify(&failed("signal 4 (unknown)"), "RIFE_PROBE:load\n"); - assert_eq!(sigill.reason, Some(RifeReason::UnsupportedCpu)); - let win = failed("exit code -1073741795 — illegal instruction: a plugin used a CPU feature"); - assert_eq!(classify(&win, "RIFE_PROBE:load\n").reason, Some(RifeReason::UnsupportedCpu)); - } - - #[test] - fn py_str_escapes_windows_paths_and_quotes() { - assert_eq!(py_str(Path::new(r"C:\Users\o'brien\rife")), r"'C:\\Users\\o\'brien\\rife'"); - assert_eq!(py_str(Path::new("/a/b")), "'/a/b'"); - } - - #[test] - fn probe_script_preloads_only_where_a_preload_exists() { - let mac = RifeFiles::in_vapoursynth_dir(Path::new("/v"), "macos-arm64"); - let s = probe_script(&mac, "macos-arm64"); - let preload = py_str(mac.preload.as_deref().unwrap()); - assert!(s.contains(&format!("ctypes.CDLL({preload}, mode=ctypes.RTLD_GLOBAL)"))); - let win = RifeFiles::in_vapoursynth_dir(Path::new("C:/v"), "windows-x64"); - let s = probe_script(&win, "windows-x64"); - assert!(s.contains("if None:")); - assert!(s.contains("'vcomp140.dll'")); - } -} diff --git a/worker/src/script_generator.rs b/worker/src/script_generator.rs index a0cd72f5..4399806f 100644 --- a/worker/src/script_generator.rs +++ b/worker/src/script_generator.rs @@ -8,6 +8,7 @@ use std::path::{Path, PathBuf}; use anyhow::{Context, Result}; +use crate::gpu_plugin::{py_str, GpuPluginFiles}; use crate::models::{ AntiAliasMethod, GrainMethod, VideoJob, ProcessingPipeline, NoiseReductionMethod, UpscaleMethod, CCD_REFERENCE_HEIGHT, @@ -65,7 +66,17 @@ pub struct ScriptGenerator { /// of producing a script that would segfault without a Vulkan device. /// Defaults to "not checked"; callers set it from /// `DependencyLocator::rife_runtime()`. - rife: std::result::Result, + rife: std::result::Result, + /// The same for AI upscaling (VSNCNN with Real-ESRGAN / Real-CUGAN). + upscale: std::result::Result, + /// The OpenCL `device_type` the KNLMeans noise-reduction method runs on, or + /// the message explaining why it cannot run here. Like RIFE, a job choosing + /// it fails with that message rather than with KNLMeansCL's own + /// `CL_INVALID_VALUE`. Defaults to "not checked"; callers set it from + /// `DependencyLocator::knlm_runtime()`. (`knlm_available` above is a + /// different question: whether QTGMC's KNLMeansCL call, which havsfunc makes + /// with the plugin's default device, works.) + knlm: std::result::Result, } /// Parameters for preview script generation. @@ -98,6 +109,8 @@ impl ScriptGenerator { opencl_available: true, knlm_available: true, rife: Err("RIFE was not checked for this job.".to_string()), + upscale: Err("AI upscaling was not checked for this job.".to_string()), + knlm: Err("KNLMeansCL was not checked for this job.".to_string()), }) } @@ -117,25 +130,48 @@ impl ScriptGenerator { } /// Set RIFE's files, or why it cannot run here (see `self.rife`). - pub fn with_rife(mut self, rife: std::result::Result) -> Self { + pub fn with_rife(mut self, rife: std::result::Result) -> Self { self.rife = rife; self } - /// Refuse a job that chose RIFE on a machine where it cannot run. - fn ensure_rife_ready(&self, pipeline: &ProcessingPipeline) -> Result<()> { + /// Set AI upscaling's files, or why it cannot run here (see `self.upscale`). + pub fn with_upscale(mut self, upscale: std::result::Result) -> Self { + self.upscale = upscale; + self + } + + /// Set the device KNLMeans runs on, or why it cannot run (see `self.knlm`). + pub fn with_knlm(mut self, knlm: std::result::Result) -> Self { + self.knlm = knlm; + self + } + + /// Refuse a job that chose a GPU-only method (RIFE, AI upscaling, KNLMeans) on a machine + /// where it cannot run, with the reason the app also shows. + fn ensure_methods_ready(&self, pipeline: &ProcessingPipeline) -> Result<()> { if pipeline.frame_rate.enabled && pipeline.frame_rate.method == FrameRateMethod::Rife { if let Err(message) = &self.rife { anyhow::bail!("{message}"); } } + if pipeline.uses_ai_upscale() { + if let Err(message) = &self.upscale { + anyhow::bail!("{message}"); + } + } + if pipeline.uses_knlm() { + if let Err(message) = &self.knlm { + anyhow::bail!("{message}"); + } + } Ok(()) } /// Generate a .vpy script file for the given job. pub fn generate(&self, job: &VideoJob) -> Result { let pipeline = job.effective_pipeline(); - self.ensure_rife_ready(&pipeline)?; + self.ensure_methods_ready(&pipeline)?; let script = self.substitute_parameters(&self.template, job, &pipeline, &job.input_path); @@ -153,7 +189,7 @@ impl ScriptGenerator { /// Returns the path to the generated script. pub fn generate_preview(&self, job: &VideoJob, preview_params: &PreviewParams) -> Result { let pipeline = job.effective_pipeline(); - self.ensure_rife_ready(&pipeline)?; + self.ensure_methods_ready(&pipeline)?; // Start with preview template and substitute preview-specific params let mut script = self.preview_template.clone(); @@ -741,6 +777,32 @@ impl ScriptGenerator { script = remove_block("{{#SPOTLESS}}", "{{/SPOTLESS}}", script); } + // ==================================================================== + // DESCALE PASS + // ==================================================================== + let ds = &pipeline.descale; + if ds.enabled { + script = script.replace("{{#DESCALE}}", ""); + script = script.replace("{{/DESCALE}}", ""); + let (inverse, forward) = ds.kernel_calls(); + script = script.replace("{{DESCALE_INVERSE}}", &inverse); + script = script.replace("{{DESCALE_FORWARD}}", &forward); + script = script.replace("{{DESCALE_HEIGHT}}", &ds.effective_native_height().to_string()); + script = script.replace("{{DESCALE_WIDTH}}", &ds.native_width.max(0).to_string()); + if ds.protect_detail { + script = script.replace("{{#DESCALE_PROTECT}}", ""); + script = script.replace("{{/DESCALE_PROTECT}}", ""); + script = script.replace( + "{{DESCALE_THRESHOLD}}", + &format_double(ds.effective_detail_threshold()), + ); + } else { + script = remove_block("{{#DESCALE_PROTECT}}", "{{/DESCALE_PROTECT}}", script); + } + } else { + script = remove_block("{{#DESCALE}}", "{{/DESCALE}}", script); + } + // ==================================================================== // NOISE REDUCTION PASS // ==================================================================== @@ -761,13 +823,52 @@ impl ScriptGenerator { script = script.replace("{{#NOISE_REDUCTION}}", ""); script = script.replace("{{/NOISE_REDUCTION}}", ""); + // BM3D and KNLMeans are removed here rather than in every other + // method's arm below. + if nr.method != NoiseReductionMethod::Bm3d { + script = remove_block("{{#NR_BM3D}}", "{{/NR_BM3D}}", script); + } + if nr.method != NoiseReductionMethod::KnlMeans { + script = remove_block("{{#NR_KNLM}}", "{{/NR_KNLM}}", script); + } + match nr.method { + NoiseReductionMethod::Bm3d | NoiseReductionMethod::KnlMeans => { + for other in [ + "NR_SMDEGRAIN", "NR_MCTD", "NR_MCDEGRAINSHARP", "NR_CTMF", + "NR_FLUXSMOOTH_T", "NR_FLUXSMOOTH_ST", "NR_STPRESSO", "NR_MCLEAN", + "NR_TD2", "NR_DFTTEST", "NR_FFT3D", "NR_TTEMPSMOOTH", + ] { + script = remove_block(&format!("{{{{#{other}}}}}"), &format!("{{{{/{other}}}}}"), script); + } + if nr.method == NoiseReductionMethod::Bm3d { + let (luma, chroma) = nr.bm3d_effective_sigmas(); + script = script.replace("{{#NR_BM3D}}", ""); + script = script.replace("{{/NR_BM3D}}", ""); + script = script.replace("{{NR_BM3D_SIGMA}}", &format!("{luma:.2}")); + script = script.replace("{{NR_BM3D_CHROMA_SIGMA}}", &format!("{chroma:.2}")); + script = script.replace("{{NR_BM3D_RADIUS}}", &nr.bm3d_effective_radius().to_string()); + script = script.replace("{{NR_BM3D_FINAL}}", if nr.bm3d_final { "True" } else { "False" }); + script = script.replace("{{NR_BM3D_PROFILE}}", nr.bm3d_effective_profile()); + } else { + let (h, d, a, s) = nr.knlm_effective(); + // ensure_methods_ready has already refused a job without one. + let device = self.knlm.as_deref().unwrap_or("gpu"); + script = script.replace("{{#NR_KNLM}}", ""); + script = script.replace("{{/NR_KNLM}}", ""); + script = script.replace("{{NR_KNLM_H}}", &format!("{h:.2}")); + script = script.replace("{{NR_KNLM_D}}", &d.to_string()); + script = script.replace("{{NR_KNLM_A}}", &a.to_string()); + script = script.replace("{{NR_KNLM_S}}", &s.to_string()); + script = script.replace("{{NR_KNLM_CHROMA}}", if nr.knlm_chroma { "True" } else { "False" }); + script = script.replace("{{NR_KNLM_DEVICE}}", device); + } + } NoiseReductionMethod::SmDegrain => { script = script.replace("{{#NR_SMDEGRAIN}}", ""); script = script.replace("{{/NR_SMDEGRAIN}}", ""); script = remove_block("{{#NR_MCTD}}", "{{/NR_MCTD}}", script); script = remove_block("{{#NR_MCDEGRAINSHARP}}", "{{/NR_MCDEGRAINSHARP}}", script); - script = remove_block("{{#NR_BM3D}}", "{{/NR_BM3D}}", script); script = remove_block("{{#NR_CTMF}}", "{{/NR_CTMF}}", script); script = remove_block("{{#NR_FLUXSMOOTH_T}}", "{{/NR_FLUXSMOOTH_T}}", script); script = remove_block("{{#NR_FLUXSMOOTH_ST}}", "{{/NR_FLUXSMOOTH_ST}}", script); @@ -790,7 +891,6 @@ impl ScriptGenerator { script = script.replace("{{#NR_MCTD}}", ""); script = script.replace("{{/NR_MCTD}}", ""); script = remove_block("{{#NR_MCDEGRAINSHARP}}", "{{/NR_MCDEGRAINSHARP}}", script); - script = remove_block("{{#NR_BM3D}}", "{{/NR_BM3D}}", script); script = remove_block("{{#NR_CTMF}}", "{{/NR_CTMF}}", script); script = remove_block("{{#NR_FLUXSMOOTH_T}}", "{{/NR_FLUXSMOOTH_T}}", script); script = remove_block("{{#NR_FLUXSMOOTH_ST}}", "{{/NR_FLUXSMOOTH_ST}}", script); @@ -810,7 +910,6 @@ impl ScriptGenerator { script = remove_block("{{#NR_MCTD}}", "{{/NR_MCTD}}", script); script = script.replace("{{#NR_MCDEGRAINSHARP}}", ""); script = script.replace("{{/NR_MCDEGRAINSHARP}}", ""); - script = remove_block("{{#NR_BM3D}}", "{{/NR_BM3D}}", script); script = remove_block("{{#NR_CTMF}}", "{{/NR_CTMF}}", script); script = remove_block("{{#NR_FLUXSMOOTH_T}}", "{{/NR_FLUXSMOOTH_T}}", script); script = remove_block("{{#NR_FLUXSMOOTH_ST}}", "{{/NR_FLUXSMOOTH_ST}}", script); @@ -856,7 +955,6 @@ impl ScriptGenerator { script = remove_block("{{#NR_SMDEGRAIN}}", "{{/NR_SMDEGRAIN}}", script); script = remove_block("{{#NR_MCTD}}", "{{/NR_MCTD}}", script); script = remove_block("{{#NR_MCDEGRAINSHARP}}", "{{/NR_MCDEGRAINSHARP}}", script); - script = remove_block("{{#NR_BM3D}}", "{{/NR_BM3D}}", script); script = remove_block("{{#NR_CTMF}}", "{{/NR_CTMF}}", script); script = remove_block("{{#NR_FLUXSMOOTH_T}}", "{{/NR_FLUXSMOOTH_T}}", script); script = remove_block("{{#NR_FLUXSMOOTH_ST}}", "{{/NR_FLUXSMOOTH_ST}}", script); @@ -887,7 +985,6 @@ impl ScriptGenerator { script = remove_block("{{#NR_SMDEGRAIN}}", "{{/NR_SMDEGRAIN}}", script); script = remove_block("{{#NR_MCTD}}", "{{/NR_MCTD}}", script); script = remove_block("{{#NR_MCDEGRAINSHARP}}", "{{/NR_MCDEGRAINSHARP}}", script); - script = remove_block("{{#NR_BM3D}}", "{{/NR_BM3D}}", script); script = remove_block("{{#NR_CTMF}}", "{{/NR_CTMF}}", script); script = remove_block("{{#NR_FLUXSMOOTH_T}}", "{{/NR_FLUXSMOOTH_T}}", script); script = remove_block("{{#NR_FLUXSMOOTH_ST}}", "{{/NR_FLUXSMOOTH_ST}}", script); @@ -919,7 +1016,6 @@ impl ScriptGenerator { script = remove_block("{{#NR_SMDEGRAIN}}", "{{/NR_SMDEGRAIN}}", script); script = remove_block("{{#NR_MCTD}}", "{{/NR_MCTD}}", script); script = remove_block("{{#NR_MCDEGRAINSHARP}}", "{{/NR_MCDEGRAINSHARP}}", script); - script = remove_block("{{#NR_BM3D}}", "{{/NR_BM3D}}", script); script = remove_block("{{#NR_CTMF}}", "{{/NR_CTMF}}", script); script = remove_block("{{#NR_FLUXSMOOTH_T}}", "{{/NR_FLUXSMOOTH_T}}", script); script = remove_block("{{#NR_FLUXSMOOTH_ST}}", "{{/NR_FLUXSMOOTH_ST}}", script); @@ -954,7 +1050,6 @@ impl ScriptGenerator { script = remove_block("{{#NR_SMDEGRAIN}}", "{{/NR_SMDEGRAIN}}", script); script = remove_block("{{#NR_MCTD}}", "{{/NR_MCTD}}", script); script = remove_block("{{#NR_MCDEGRAINSHARP}}", "{{/NR_MCDEGRAINSHARP}}", script); - script = remove_block("{{#NR_BM3D}}", "{{/NR_BM3D}}", script); script = remove_block("{{#NR_CTMF}}", "{{/NR_CTMF}}", script); script = remove_block("{{#NR_DFTTEST}}", "{{/NR_DFTTEST}}", script); script = remove_block("{{#NR_FFT3D}}", "{{/NR_FFT3D}}", script); @@ -990,7 +1085,6 @@ impl ScriptGenerator { script = remove_block("{{#NR_SMDEGRAIN}}", "{{/NR_SMDEGRAIN}}", script); script = remove_block("{{#NR_MCTD}}", "{{/NR_MCTD}}", script); script = remove_block("{{#NR_MCDEGRAINSHARP}}", "{{/NR_MCDEGRAINSHARP}}", script); - script = remove_block("{{#NR_BM3D}}", "{{/NR_BM3D}}", script); script = remove_block("{{#NR_CTMF}}", "{{/NR_CTMF}}", script); script = remove_block("{{#NR_MCLEAN}}", "{{/NR_MCLEAN}}", script); script = remove_block("{{#NR_TD2}}", "{{/NR_TD2}}", script); @@ -1010,7 +1104,6 @@ impl ScriptGenerator { script = remove_block("{{#NR_SMDEGRAIN}}", "{{/NR_SMDEGRAIN}}", script); script = remove_block("{{#NR_MCTD}}", "{{/NR_MCTD}}", script); script = remove_block("{{#NR_MCDEGRAINSHARP}}", "{{/NR_MCDEGRAINSHARP}}", script); - script = remove_block("{{#NR_BM3D}}", "{{/NR_BM3D}}", script); script = remove_block("{{#NR_DFTTEST}}", "{{/NR_DFTTEST}}", script); script = remove_block("{{#NR_FFT3D}}", "{{/NR_FFT3D}}", script); script = remove_block("{{#NR_TTEMPSMOOTH}}", "{{/NR_TTEMPSMOOTH}}", script); @@ -1031,7 +1124,6 @@ impl ScriptGenerator { script = remove_block("{{#NR_SMDEGRAIN}}", "{{/NR_SMDEGRAIN}}", script); script = remove_block("{{#NR_MCTD}}", "{{/NR_MCTD}}", script); script = remove_block("{{#NR_MCDEGRAINSHARP}}", "{{/NR_MCDEGRAINSHARP}}", script); - script = remove_block("{{#NR_BM3D}}", "{{/NR_BM3D}}", script); script = remove_block("{{#NR_DFTTEST}}", "{{/NR_DFTTEST}}", script); script = remove_block("{{#NR_FFT3D}}", "{{/NR_FFT3D}}", script); script = remove_block("{{#NR_TTEMPSMOOTH}}", "{{/NR_TTEMPSMOOTH}}", script); @@ -1055,7 +1147,6 @@ impl ScriptGenerator { script = remove_block("{{#NR_SMDEGRAIN}}", "{{/NR_SMDEGRAIN}}", script); script = remove_block("{{#NR_MCTD}}", "{{/NR_MCTD}}", script); script = remove_block("{{#NR_MCDEGRAINSHARP}}", "{{/NR_MCDEGRAINSHARP}}", script); - script = remove_block("{{#NR_BM3D}}", "{{/NR_BM3D}}", script); script = remove_block("{{#NR_DFTTEST}}", "{{/NR_DFTTEST}}", script); script = remove_block("{{#NR_FFT3D}}", "{{/NR_FFT3D}}", script); script = remove_block("{{#NR_TTEMPSMOOTH}}", "{{/NR_TTEMPSMOOTH}}", script); @@ -1079,7 +1170,6 @@ impl ScriptGenerator { script = remove_block("{{#NR_SMDEGRAIN}}", "{{/NR_SMDEGRAIN}}", script); script = remove_block("{{#NR_MCTD}}", "{{/NR_MCTD}}", script); script = remove_block("{{#NR_MCDEGRAINSHARP}}", "{{/NR_MCDEGRAINSHARP}}", script); - script = remove_block("{{#NR_BM3D}}", "{{/NR_BM3D}}", script); script = remove_block("{{#NR_CTMF}}", "{{/NR_CTMF}}", script); script = remove_block("{{#NR_FLUXSMOOTH_T}}", "{{/NR_FLUXSMOOTH_T}}", script); script = remove_block("{{#NR_FLUXSMOOTH_ST}}", "{{/NR_FLUXSMOOTH_ST}}", script); @@ -1756,9 +1846,9 @@ impl ScriptGenerator { // files, so the fallbacks here are never reached by a job. let (plugin, model, preload) = match &self.rife { Ok(f) => ( - crate::rife::py_str(&f.plugin), - crate::rife::py_str(&f.model_dir), - f.preload.as_deref().map(crate::rife::py_str).unwrap_or_else(|| "None".into()), + py_str(&f.plugin), + py_str(&f.model_dir), + f.preload.as_deref().map(py_str).unwrap_or_else(|| "None".into()), ), Err(_) => ("''".into(), "''".into(), "None".into()), }; @@ -1939,15 +2029,50 @@ impl ScriptGenerator { script = script.replace("{{/RESIZE_INTEGER_UPSCALE}}", ""); script = script.replace("{{UPSCALE_FACTOR}}", &resize.upscale_factor.to_string()); - // Two of the three methods are edge-directed and share the - // doubling loop and shift correction; Spline36 is plain - // resampling and skips all of it. - let edge_directed = !matches!(resize.upscale_method, UpscaleMethod::Spline36); + // Two of the methods are edge-directed and share the doubling + // loop and shift correction; Spline36 is plain resampling and + // the AI methods run a network, so they skip all of it. + let edge_directed = matches!( + resize.upscale_method, + UpscaleMethod::Nnedi3Rpow2 | UpscaleMethod::Eedi3Rpow2 + ); + if resize.upscale_method.is_ai() { + script = remove_block("{{#UPSCALE_EDGE_DIRECTED}}", "{{/UPSCALE_EDGE_DIRECTED}}", script); + script = remove_block("{{#UPSCALE_SPLINE36}}", "{{/UPSCALE_SPLINE36}}", script); + script = script.replace("{{#UPSCALE_AI}}", ""); + script = script.replace("{{/UPSCALE_AI}}", ""); + // generate()/generate_preview() refuse such a job without + // files, so the fallbacks here are never reached by a job. + let (plugin, preload, models) = match &self.upscale { + Ok(f) => ( + py_str(&f.plugin), + f.preload.as_deref().map(py_str).unwrap_or_else(|| "None".into()), + resize + .ai_upscale_models() + .iter() + .map(|m| py_str(&f.model_dir.join(m))) + .collect::>() + .join(", "), + ), + Err(_) => ("''".into(), "None".into(), String::new()), + }; + script = script.replace("{{AI_PLUGIN}}", &plugin); + script = script.replace("{{AI_PRELOAD}}", &preload); + script = script.replace("{{AI_MODELS}}", &models); + let color = job.color_metadata(); + script = script.replace("{{AI_MATRIX}}", color.zimg_matrix(job.input_height)); + script = script.replace( + "{{AI_RANGE}}", + if color.is_full_range() { "full" } else { "limited" }, + ); + } else { + script = remove_block("{{#UPSCALE_AI}}", "{{/UPSCALE_AI}}", script); + } if edge_directed { script = script.replace("{{#UPSCALE_EDGE_DIRECTED}}", ""); script = script.replace("{{/UPSCALE_EDGE_DIRECTED}}", ""); script = remove_block("{{#UPSCALE_SPLINE36}}", "{{/UPSCALE_SPLINE36}}", script); - } else { + } else if resize.upscale_method == UpscaleMethod::Spline36 { script = remove_block("{{#UPSCALE_EDGE_DIRECTED}}", "{{/UPSCALE_EDGE_DIRECTED}}", script); script = script.replace("{{#UPSCALE_SPLINE36}}", ""); script = script.replace("{{/UPSCALE_SPLINE36}}", ""); @@ -1970,7 +2095,7 @@ impl ScriptGenerator { script = process_optional_int("UPSCALE_NRAD", resize.upscale_nrad, script); script = process_optional_int("UPSCALE_MDIS", resize.upscale_mdis, script); } - UpscaleMethod::Spline36 => { + UpscaleMethod::Spline36 | UpscaleMethod::RealEsrgan | UpscaleMethod::RealCugan => { script = remove_block("{{#UPSCALE_NNEDI3}}", "{{/UPSCALE_NNEDI3}}", script); script = remove_block("{{#UPSCALE_EEDI3}}", "{{/UPSCALE_EEDI3}}", script); } diff --git a/worker/templates/pipeline_template.vpy b/worker/templates/pipeline_template.vpy index f0d09465..620e5145 100644 --- a/worker/templates/pipeline_template.vpy +++ b/worker/templates/pipeline_template.vpy @@ -602,6 +602,57 @@ clip = _remove_dirt( {{/SPOTLESS_REMOVEDIRT}} {{/SPOTLESS}} +{{#DESCALE}} +# Descale: undo the studio's upscale (luma), then upscale again with NNEDI3. +# Most modern anime is drawn below 1080p and upscaled with a plain kernel; +# with the right native size and kernel, descale inverts that resample exactly +# and NNEDI3 redraws far cleaner lines than the studio's kernel gave. Chroma is +# left alone (it was stored below the luma resolution to begin with) and the +# output keeps the source's size, so Crop & Resize can still upscale it. +_ds_src = clip +_ds_fmt = clip.format +_ds_h = {{DESCALE_HEIGHT}} +if clip.height <= _ds_h: + raise vs.Error(f"Descale: the native height ({_ds_h}) must be below the source's ({clip.height}).") +_ds_w = {{DESCALE_WIDTH}} or int(round(_ds_h * clip.width / clip.height)) +_ds_y = core.resize.Point(core.std.ShufflePlanes(clip, 0, vs.GRAY), format=vs.GRAYS) +def _ds_descale(c, w, h): + return {{DESCALE_INVERSE}} +def _ds_resample(c, w, h): + return {{DESCALE_FORWARD}} +_ds_native = _ds_descale(_ds_y, _ds_w, _ds_h) +# NNEDI3 doubling (height, transpose, height, transpose back) until the picture +# is at least the source size, then one resample down to it. Each doubling +# leaves half an output pixel of offset, doubled by every later doubling; it is +# folded into that one resample. +_ds_up, _ds_shift = _ds_native, 0.0 +while _ds_up.width < clip.width or _ds_up.height < clip.height: + _ds_up = _nnedi3(_ds_up, field=1, dh=True, nsize=4, nns=4, qual=2) + _ds_up = core.std.Transpose(_ds_up) + _ds_up = _nnedi3(_ds_up, field=1, dh=True, nsize=4, nns=4, qual=2) + _ds_up = core.std.Transpose(_ds_up) + _ds_shift = _ds_shift * 2 + 0.5 +_ds_up = core.resize.Spline36(_ds_up, clip.width, clip.height, src_left=-_ds_shift, src_top=-_ds_shift) +{{#DESCALE_PROTECT}} +# Detail drawn at full resolution (credits, text, overlays) was never upscaled, +# so re-applying the kernel to the descaled picture does not reproduce it. +# Wherever that misses by more than the threshold (8-bit levels), keep the +# source, over a slightly grown and softened mask. +_ds_error = _expr([_ds_y, _ds_resample(_ds_native, clip.width, clip.height)], "x y - abs") +_ds_mask = core.std.Binarize(_ds_error, {{DESCALE_THRESHOLD}} / 255.0) +_ds_mask = core.std.Inflate(core.std.Maximum(core.std.Maximum(_ds_mask))) +_ds_up = core.std.MaskedMerge(_ds_up, _ds_y, _ds_mask) +{{/DESCALE_PROTECT}} +_ds_luma = core.resize.Point( + _ds_up, format=_ds_fmt.replace(color_family=vs.GRAY, subsampling_w=0, subsampling_h=0).id, + dither_type="error_diffusion") +if _ds_fmt.num_planes > 1: + clip = core.std.ShufflePlanes([_ds_luma, _ds_src], [0, 1, 2], _ds_fmt.color_family) +else: + clip = _ds_luma +clip = core.std.CopyFrameProps(clip, _ds_src) +{{/DESCALE}} + # ============================================================================ # PASS 5: NOISE REDUCTION # ============================================================================ @@ -838,17 +889,47 @@ clip = core.mv.Degrain3( {{/NR_MCDEGRAINSHARP}} {{#NR_BM3D}} -# BM3D noise reduction -clip = mvf.BM3D( - clip, -{{#NR_BM3D_SIGMA}} - sigma={{NR_BM3D_SIGMA}}, -{{/NR_BM3D_SIGMA}} -{{#NR_BM3D_RADIUS}} - radius={{NR_BM3D_RADIUS}}, -{{/NR_BM3D_RADIUS}} -) +# BM3D: groups similar blocks across the frame (and with a radius, across +# 2r+1 frames: V-BM3D) and filters each group together, so flat areas come out +# clean without smearing line art. sigma is in 8-bit units at every depth. +# Works in 16-bit: VAggregate always returns 16-bit and VFinal needs its basic +# estimate in the same format as its input, so every step runs there and the +# source format is restored after, with dither. Each plane is filtered as its +# own GRAY clip: BM3D refuses subsampled chroma, and this denoises 4:2:0 chroma +# at its own resolution instead of resampling it to 4:4:4 and back. +_bm3d_fmt = clip.format +_bm3d_radius = {{NR_BM3D_RADIUS}} +_bm3d_final = {{NR_BM3D_FINAL}} +_bm3d_profile = "{{NR_BM3D_PROFILE}}" +def _bm3d_plane(p, sigma): + if _bm3d_radius > 0: + r = _bm3d_radius + ref = core.bm3d.VAggregate(core.bm3d.VBasic(p, sigma=sigma, radius=r, profile=_bm3d_profile), radius=r) + if _bm3d_final: + ref = core.bm3d.VAggregate(core.bm3d.VFinal(p, ref, sigma=sigma, radius=r, profile=_bm3d_profile), radius=r) + return ref + ref = core.bm3d.Basic(p, sigma=sigma, profile=_bm3d_profile) + return core.bm3d.Final(p, ref, sigma=sigma, profile=_bm3d_profile) if _bm3d_final else ref +_bm3d_work = core.resize.Point(clip, format=_bm3d_fmt.replace(bits_per_sample=16, sample_type=vs.INTEGER).id) +_bm3d_planes = [core.std.ShufflePlanes(_bm3d_work, i, vs.GRAY) for i in range(_bm3d_fmt.num_planes)] +_bm3d_sigmas = [{{NR_BM3D_SIGMA}}] + [{{NR_BM3D_CHROMA_SIGMA}}] * (_bm3d_fmt.num_planes - 1) +_bm3d_planes = [_bm3d_plane(p, s) if s > 0 else p for p, s in zip(_bm3d_planes, _bm3d_sigmas)] +if _bm3d_fmt.num_planes > 1: + _bm3d_work = core.std.ShufflePlanes(_bm3d_planes, [0, 0, 0], _bm3d_fmt.color_family) +else: + _bm3d_work = _bm3d_planes[0] +clip = core.resize.Point(_bm3d_work, format=_bm3d_fmt.id, dither_type="error_diffusion") {{/NR_BM3D}} +{{#NR_KNLM}} +# KNLMeansCL: non-local means on the GPU (OpenCL). d = frames each side, +# a = search radius, s = similarity radius, h = strength. Runs on the device the +# worker's probe found working ({{NR_KNLM_DEVICE}}): the plugin's own default, +# "auto", fails on Apple Silicon. 4:2:0 cannot take channels="YUV", so luma and +# chroma are separate calls. +clip = core.knlm.KNLMeansCL(clip, d={{NR_KNLM_D}}, a={{NR_KNLM_A}}, s={{NR_KNLM_S}}, h={{NR_KNLM_H}}, channels="Y", device_type="{{NR_KNLM_DEVICE}}") +if {{NR_KNLM_CHROMA}} and clip.format.color_family == vs.YUV: + clip = core.knlm.KNLMeansCL(clip, d={{NR_KNLM_D}}, a={{NR_KNLM_A}}, s={{NR_KNLM_S}}, h={{NR_KNLM_H}}, channels="UV", device_type="{{NR_KNLM_DEVICE}}") +{{/NR_KNLM}} {{#NR_MCLEAN}} # mClean — denoise, then put detail and grain back, so the result does not look # plastic. The only method here that is a goal rather than a mechanism. @@ -1910,6 +1991,52 @@ clip = _upscale_fix_shift(clip, _upscale_shift) # Plain resampling at an integer factor — no edge direction, just fast. clip = core.resize.Spline36(clip, width=clip.width * {{UPSCALE_FACTOR}}, height=clip.height * {{UPSCALE_FACTOR}}) {{/UPSCALE_SPLINE36}} +{{#UPSCALE_AI}} +# AI upscaling: Real-ESRGAN / Real-CUGAN through vs-mlrt's VSNCNN (ncnn on +# Vulkan), loaded explicitly from vapoursynth/ncnn/ (never autoloaded) after +# MoltenVK on macOS, exactly as RIFE is. The networks take float RGB, so the +# source's matrix and range convert there and back, and the result is dithered +# to the source format. Above SD the frame runs in overlapping 512-pixel tiles: +# whole-frame Real-CUGAN at 1080p ran out of GPU memory on a 16 GB Mac. An odd +# size is padded to even by repeating the edge (CUGAN works in 2-pixel steps) +# and cropped after. fp16 differs from fp32 by at most 0.3/255 and is ~3x faster. +import ctypes as _ai_ctypes +_ai_preload = {{AI_PRELOAD}} +if _ai_preload: + _ai_ctypes.CDLL(_ai_preload, mode=_ai_ctypes.RTLD_GLOBAL) +if not hasattr(core, "ncnn"): + core.std.LoadPlugin({{AI_PLUGIN}}) +_ai_src = clip +_ai_fmt = clip.format +if _ai_fmt.color_family == vs.YUV: + _ai_rgb = core.resize.Spline36(clip, format=vs.RGBS, matrix_in_s="{{AI_MATRIX}}", range_in_s="{{AI_RANGE}}") +elif _ai_fmt.color_family == vs.GRAY: + _ai_rgb = core.std.ShufflePlanes( + core.resize.Point(clip, format=vs.GRAYS, range_in_s="{{AI_RANGE}}"), [0, 0, 0], vs.RGB) +else: + _ai_rgb = core.resize.Point(clip, format=vs.RGBS) +_ai_pad_w, _ai_pad_h = _ai_rgb.width % 2, _ai_rgb.height % 2 +if _ai_pad_w or _ai_pad_h: + # Point resampling past the source edge repeats the last row/column. + _ai_rgb = core.resize.Point( + _ai_rgb, width=_ai_rgb.width + _ai_pad_w, height=_ai_rgb.height + _ai_pad_h, + src_width=_ai_rgb.width + _ai_pad_w, src_height=_ai_rgb.height + _ai_pad_h) +_ai_in_w = _ai_rgb.width +for _ai_model in [{{AI_MODELS}}]: + _ai_tiles = {} if _ai_rgb.width * _ai_rgb.height <= 720 * 576 else dict(tilesize=[512, 512], overlap=[16, 16]) + _ai_rgb = core.ncnn.Model(_ai_rgb, network_path=_ai_model, fp16=True, num_streams=2, **_ai_tiles) +_ai_scale = _ai_rgb.width // _ai_in_w +if _ai_pad_w or _ai_pad_h: + _ai_rgb = core.std.Crop(_ai_rgb, right=_ai_pad_w * _ai_scale, bottom=_ai_pad_h * _ai_scale) +if _ai_fmt.color_family == vs.YUV: + clip = core.resize.Spline36(_ai_rgb, format=_ai_fmt.id, matrix_s="{{AI_MATRIX}}", range_s="{{AI_RANGE}}", dither_type="error_diffusion") +elif _ai_fmt.color_family == vs.GRAY: + clip = core.resize.Point(core.std.ShufflePlanes(_ai_rgb, 1, vs.GRAY), format=_ai_fmt.id, range_s="{{AI_RANGE}}", dither_type="error_diffusion") +else: + clip = core.resize.Point(_ai_rgb, format=_ai_fmt.id, dither_type="error_diffusion") +# The network drops frame properties; field order, matrix and the rest are the source's. +clip = core.std.CopyFrameProps(clip, _ai_src) +{{/UPSCALE_AI}} {{/RESIZE_INTEGER_UPSCALE}} {{#RESIZE_STANDARD}} diff --git a/worker/templates/preview_template.vpy b/worker/templates/preview_template.vpy index 9e27de19..6ac71b10 100644 --- a/worker/templates/preview_template.vpy +++ b/worker/templates/preview_template.vpy @@ -552,6 +552,57 @@ clip = _remove_dirt( {{/SPOTLESS_REMOVEDIRT}} {{/SPOTLESS}} +{{#DESCALE}} +# Descale: undo the studio's upscale (luma), then upscale again with NNEDI3. +# Most modern anime is drawn below 1080p and upscaled with a plain kernel; +# with the right native size and kernel, descale inverts that resample exactly +# and NNEDI3 redraws far cleaner lines than the studio's kernel gave. Chroma is +# left alone (it was stored below the luma resolution to begin with) and the +# output keeps the source's size, so Crop & Resize can still upscale it. +_ds_src = clip +_ds_fmt = clip.format +_ds_h = {{DESCALE_HEIGHT}} +if clip.height <= _ds_h: + raise vs.Error(f"Descale: the native height ({_ds_h}) must be below the source's ({clip.height}).") +_ds_w = {{DESCALE_WIDTH}} or int(round(_ds_h * clip.width / clip.height)) +_ds_y = core.resize.Point(core.std.ShufflePlanes(clip, 0, vs.GRAY), format=vs.GRAYS) +def _ds_descale(c, w, h): + return {{DESCALE_INVERSE}} +def _ds_resample(c, w, h): + return {{DESCALE_FORWARD}} +_ds_native = _ds_descale(_ds_y, _ds_w, _ds_h) +# NNEDI3 doubling (height, transpose, height, transpose back) until the picture +# is at least the source size, then one resample down to it. Each doubling +# leaves half an output pixel of offset, doubled by every later doubling; it is +# folded into that one resample. +_ds_up, _ds_shift = _ds_native, 0.0 +while _ds_up.width < clip.width or _ds_up.height < clip.height: + _ds_up = _nnedi3(_ds_up, field=1, dh=True, nsize=4, nns=4, qual=2) + _ds_up = core.std.Transpose(_ds_up) + _ds_up = _nnedi3(_ds_up, field=1, dh=True, nsize=4, nns=4, qual=2) + _ds_up = core.std.Transpose(_ds_up) + _ds_shift = _ds_shift * 2 + 0.5 +_ds_up = core.resize.Spline36(_ds_up, clip.width, clip.height, src_left=-_ds_shift, src_top=-_ds_shift) +{{#DESCALE_PROTECT}} +# Detail drawn at full resolution (credits, text, overlays) was never upscaled, +# so re-applying the kernel to the descaled picture does not reproduce it. +# Wherever that misses by more than the threshold (8-bit levels), keep the +# source, over a slightly grown and softened mask. +_ds_error = _expr([_ds_y, _ds_resample(_ds_native, clip.width, clip.height)], "x y - abs") +_ds_mask = core.std.Binarize(_ds_error, {{DESCALE_THRESHOLD}} / 255.0) +_ds_mask = core.std.Inflate(core.std.Maximum(core.std.Maximum(_ds_mask))) +_ds_up = core.std.MaskedMerge(_ds_up, _ds_y, _ds_mask) +{{/DESCALE_PROTECT}} +_ds_luma = core.resize.Point( + _ds_up, format=_ds_fmt.replace(color_family=vs.GRAY, subsampling_w=0, subsampling_h=0).id, + dither_type="error_diffusion") +if _ds_fmt.num_planes > 1: + clip = core.std.ShufflePlanes([_ds_luma, _ds_src], [0, 1, 2], _ds_fmt.color_family) +else: + clip = _ds_luma +clip = core.std.CopyFrameProps(clip, _ds_src) +{{/DESCALE}} + # ============================================================================ # PASS 5: NOISE REDUCTION # ============================================================================ @@ -781,17 +832,47 @@ clip = core.mv.Degrain3( {{/NR_MCDEGRAINSHARP}} {{#NR_BM3D}} -# BM3D noise reduction -clip = mvf.BM3D( - clip, -{{#NR_BM3D_SIGMA}} - sigma={{NR_BM3D_SIGMA}}, -{{/NR_BM3D_SIGMA}} -{{#NR_BM3D_RADIUS}} - radius={{NR_BM3D_RADIUS}}, -{{/NR_BM3D_RADIUS}} -) +# BM3D: groups similar blocks across the frame (and with a radius, across +# 2r+1 frames: V-BM3D) and filters each group together, so flat areas come out +# clean without smearing line art. sigma is in 8-bit units at every depth. +# Works in 16-bit: VAggregate always returns 16-bit and VFinal needs its basic +# estimate in the same format as its input, so every step runs there and the +# source format is restored after, with dither. Each plane is filtered as its +# own GRAY clip: BM3D refuses subsampled chroma, and this denoises 4:2:0 chroma +# at its own resolution instead of resampling it to 4:4:4 and back. +_bm3d_fmt = clip.format +_bm3d_radius = {{NR_BM3D_RADIUS}} +_bm3d_final = {{NR_BM3D_FINAL}} +_bm3d_profile = "{{NR_BM3D_PROFILE}}" +def _bm3d_plane(p, sigma): + if _bm3d_radius > 0: + r = _bm3d_radius + ref = core.bm3d.VAggregate(core.bm3d.VBasic(p, sigma=sigma, radius=r, profile=_bm3d_profile), radius=r) + if _bm3d_final: + ref = core.bm3d.VAggregate(core.bm3d.VFinal(p, ref, sigma=sigma, radius=r, profile=_bm3d_profile), radius=r) + return ref + ref = core.bm3d.Basic(p, sigma=sigma, profile=_bm3d_profile) + return core.bm3d.Final(p, ref, sigma=sigma, profile=_bm3d_profile) if _bm3d_final else ref +_bm3d_work = core.resize.Point(clip, format=_bm3d_fmt.replace(bits_per_sample=16, sample_type=vs.INTEGER).id) +_bm3d_planes = [core.std.ShufflePlanes(_bm3d_work, i, vs.GRAY) for i in range(_bm3d_fmt.num_planes)] +_bm3d_sigmas = [{{NR_BM3D_SIGMA}}] + [{{NR_BM3D_CHROMA_SIGMA}}] * (_bm3d_fmt.num_planes - 1) +_bm3d_planes = [_bm3d_plane(p, s) if s > 0 else p for p, s in zip(_bm3d_planes, _bm3d_sigmas)] +if _bm3d_fmt.num_planes > 1: + _bm3d_work = core.std.ShufflePlanes(_bm3d_planes, [0, 0, 0], _bm3d_fmt.color_family) +else: + _bm3d_work = _bm3d_planes[0] +clip = core.resize.Point(_bm3d_work, format=_bm3d_fmt.id, dither_type="error_diffusion") {{/NR_BM3D}} +{{#NR_KNLM}} +# KNLMeansCL: non-local means on the GPU (OpenCL). d = frames each side, +# a = search radius, s = similarity radius, h = strength. Runs on the device the +# worker's probe found working ({{NR_KNLM_DEVICE}}): the plugin's own default, +# "auto", fails on Apple Silicon. 4:2:0 cannot take channels="YUV", so luma and +# chroma are separate calls. +clip = core.knlm.KNLMeansCL(clip, d={{NR_KNLM_D}}, a={{NR_KNLM_A}}, s={{NR_KNLM_S}}, h={{NR_KNLM_H}}, channels="Y", device_type="{{NR_KNLM_DEVICE}}") +if {{NR_KNLM_CHROMA}} and clip.format.color_family == vs.YUV: + clip = core.knlm.KNLMeansCL(clip, d={{NR_KNLM_D}}, a={{NR_KNLM_A}}, s={{NR_KNLM_S}}, h={{NR_KNLM_H}}, channels="UV", device_type="{{NR_KNLM_DEVICE}}") +{{/NR_KNLM}} {{#NR_MCLEAN}} # mClean — denoise, then put detail and grain back, so the result does not look # plastic. The only method here that is a goal rather than a mechanism. @@ -1842,6 +1923,52 @@ clip = _upscale_fix_shift(clip, _upscale_shift) # Plain resampling at an integer factor — no edge direction, just fast. clip = core.resize.Spline36(clip, width=clip.width * {{UPSCALE_FACTOR}}, height=clip.height * {{UPSCALE_FACTOR}}) {{/UPSCALE_SPLINE36}} +{{#UPSCALE_AI}} +# AI upscaling: Real-ESRGAN / Real-CUGAN through vs-mlrt's VSNCNN (ncnn on +# Vulkan), loaded explicitly from vapoursynth/ncnn/ (never autoloaded) after +# MoltenVK on macOS, exactly as RIFE is. The networks take float RGB, so the +# source's matrix and range convert there and back, and the result is dithered +# to the source format. Above SD the frame runs in overlapping 512-pixel tiles: +# whole-frame Real-CUGAN at 1080p ran out of GPU memory on a 16 GB Mac. An odd +# size is padded to even by repeating the edge (CUGAN works in 2-pixel steps) +# and cropped after. fp16 differs from fp32 by at most 0.3/255 and is ~3x faster. +import ctypes as _ai_ctypes +_ai_preload = {{AI_PRELOAD}} +if _ai_preload: + _ai_ctypes.CDLL(_ai_preload, mode=_ai_ctypes.RTLD_GLOBAL) +if not hasattr(core, "ncnn"): + core.std.LoadPlugin({{AI_PLUGIN}}) +_ai_src = clip +_ai_fmt = clip.format +if _ai_fmt.color_family == vs.YUV: + _ai_rgb = core.resize.Spline36(clip, format=vs.RGBS, matrix_in_s="{{AI_MATRIX}}", range_in_s="{{AI_RANGE}}") +elif _ai_fmt.color_family == vs.GRAY: + _ai_rgb = core.std.ShufflePlanes( + core.resize.Point(clip, format=vs.GRAYS, range_in_s="{{AI_RANGE}}"), [0, 0, 0], vs.RGB) +else: + _ai_rgb = core.resize.Point(clip, format=vs.RGBS) +_ai_pad_w, _ai_pad_h = _ai_rgb.width % 2, _ai_rgb.height % 2 +if _ai_pad_w or _ai_pad_h: + # Point resampling past the source edge repeats the last row/column. + _ai_rgb = core.resize.Point( + _ai_rgb, width=_ai_rgb.width + _ai_pad_w, height=_ai_rgb.height + _ai_pad_h, + src_width=_ai_rgb.width + _ai_pad_w, src_height=_ai_rgb.height + _ai_pad_h) +_ai_in_w = _ai_rgb.width +for _ai_model in [{{AI_MODELS}}]: + _ai_tiles = {} if _ai_rgb.width * _ai_rgb.height <= 720 * 576 else dict(tilesize=[512, 512], overlap=[16, 16]) + _ai_rgb = core.ncnn.Model(_ai_rgb, network_path=_ai_model, fp16=True, num_streams=2, **_ai_tiles) +_ai_scale = _ai_rgb.width // _ai_in_w +if _ai_pad_w or _ai_pad_h: + _ai_rgb = core.std.Crop(_ai_rgb, right=_ai_pad_w * _ai_scale, bottom=_ai_pad_h * _ai_scale) +if _ai_fmt.color_family == vs.YUV: + clip = core.resize.Spline36(_ai_rgb, format=_ai_fmt.id, matrix_s="{{AI_MATRIX}}", range_s="{{AI_RANGE}}", dither_type="error_diffusion") +elif _ai_fmt.color_family == vs.GRAY: + clip = core.resize.Point(core.std.ShufflePlanes(_ai_rgb, 1, vs.GRAY), format=_ai_fmt.id, range_s="{{AI_RANGE}}", dither_type="error_diffusion") +else: + clip = core.resize.Point(_ai_rgb, format=_ai_fmt.id, dither_type="error_diffusion") +# The network drops frame properties; field order, matrix and the rest are the source's. +clip = core.std.CopyFrameProps(clip, _ai_src) +{{/UPSCALE_AI}} {{/RESIZE_INTEGER_UPSCALE}} {{#RESIZE_STANDARD}} diff --git a/worker/tests/filter_integration_test.rs b/worker/tests/filter_integration_test.rs index 4e2c42d4..89f682b8 100644 --- a/worker/tests/filter_integration_test.rs +++ b/worker/tests/filter_integration_test.rs @@ -3605,7 +3605,10 @@ fn test_93_templates_do_not_call_std_expr_directly() { /// cases that assert something is *absent*, which is the direction that catches /// a block the generator failed to strip. fn script_text(job: &VideoJob) -> String { - let generator = ScriptGenerator::new().expect("Failed to create generator"); + // A KNLMeans job needs a device; the refusal without one is test_171's. + let generator = ScriptGenerator::new() + .expect("Failed to create generator") + .with_knlm(Ok("gpu".into())); let script_path = generator.generate(job).expect("Failed to generate script"); std::fs::read_to_string(&script_path).expect("read generated script") } @@ -4318,6 +4321,8 @@ fn test_115_every_noise_method_still_emits_only_its_own_filter() { ("fluxt", "core.flux.SmoothT("), ("fluxst", "core.flux.SmoothST("), ("stpresso", "haf.STPresso"), + ("bm3d", "core.bm3d."), + ("knlm", "core.knlm.KNLMeansCL("), ]; let methods = [ @@ -4329,6 +4334,8 @@ fn test_115_every_noise_method_still_emits_only_its_own_filter() { (NoiseReductionMethod::FluxSmoothT, "fluxt"), (NoiseReductionMethod::FluxSmoothSt, "fluxst"), (NoiseReductionMethod::StPresso, "stpresso"), + (NoiseReductionMethod::Bm3d, "bm3d"), + (NoiseReductionMethod::KnlMeans, "knlm"), ]; for (method, key) in methods { @@ -5456,7 +5463,22 @@ fn test_149_every_noise_reduction_method_emits_its_filter() { (NoiseReductionMethod::Ctmf, "core.ctmf.CTMF("), (NoiseReductionMethod::MClean, "_mClean("), (NoiseReductionMethod::TemporalDegrain2, "_TemporalDegrain2("), + (NoiseReductionMethod::Bm3d, "core.bm3d.VBasic("), + (NoiseReductionMethod::KnlMeans, "core.knlm.KNLMeansCL("), ]; + // The list above is hand-written; this match is not. A new method fails to + // compile here until it is added both here and to the list. + fn listed(m: NoiseReductionMethod) -> bool { + use NoiseReductionMethod::*; + match m { + SmDegrain | McTemporalDenoise | McDegrainSharp | DfTtest | Fft3dFilter + | TTempSmooth | FluxSmoothT | FluxSmoothSt | StPresso | Ctmf | MClean + | TemporalDegrain2 | Bm3d | KnlMeans => true, + QtgmcBuiltin => false, // denoises inside the QTGMC pass + } + } + assert_eq!(expected.len(), 14); + assert!(expected.iter().all(|(m, _)| listed(*m))); for (method, call) in expected { let mut job = create_base_job("test_149_nr_methods"); @@ -6140,8 +6162,9 @@ fn rife_job(name: &str) -> VideoJob { job } -fn rife_files() -> vapourbox_worker::rife::RifeFiles { - vapourbox_worker::rife::RifeFiles::in_vapoursynth_dir( +fn rife_files() -> vapourbox_worker::gpu_plugin::GpuPluginFiles { + vapourbox_worker::gpu_plugin::GpuPluginFiles::in_vapoursynth_dir( + vapourbox_worker::gpu_plugin::GpuPlugin::Rife, std::path::Path::new("/deps/macos-arm64/vapoursynth"), "macos-arm64", ) @@ -6164,7 +6187,7 @@ fn test_166_rife_frame_rate_emits_explicit_load_and_real_frame_passthrough() { create_output_dir(); let files = rife_files(); // Paths as the generator writes them: Path::join uses `\` on Windows. - let py = |p: &std::path::Path| vapourbox_worker::rife::py_str(p); + let py = |p: &std::path::Path| vapourbox_worker::gpu_plugin::py_str(p); let (preload, plugin, model) = (py(files.preload.as_deref().unwrap()), py(&files.plugin), py(&files.model_dir)); let generator = ScriptGenerator::new().unwrap().with_rife(Ok(files)); @@ -6243,3 +6266,290 @@ fn test_168_rife_frame_map_is_an_exact_count_retime() { assert_eq!(p.output_count(40), want, "{target:?}"); } } + +fn nr_job(name: &str, nr: NoiseReductionParameters) -> VideoJob { + let mut job = create_base_job(name); + job.qtgmc_parameters.enabled = false; + job.processing_pipeline = Some(ProcessingPipeline { + deinterlace: QTGMCParameters { enabled: false, ..Default::default() }, + noise_reduction: NoiseReductionParameters { enabled: true, ..nr }, + ..ProcessingPipeline::default() + }); + job +} + +/// BM3D's values reach both scripts, in the 16-bit per-plane block. +#[test] +fn test_169_bm3d_noise_reduction() { + create_output_dir(); + let job = nr_job("test_169_bm3d", NoiseReductionParameters { + method: NoiseReductionMethod::Bm3d, + bm3d_sigma: 4.5, + bm3d_chroma_sigma: 2.0, + bm3d_radius: 2, + bm3d_final: false, + bm3d_profile: "high".into(), + ..Default::default() + }); + let (encode, preview) = both_scripts_with(&ScriptGenerator::new().unwrap(), &job).unwrap(); + for (name, script) in [("encode", &encode), ("preview", &preview)] { + for want in [ + "_bm3d_radius = 2", + "_bm3d_final = False", + "_bm3d_profile = \"high\"", + "_bm3d_sigmas = [4.50] + [2.00]", + "core.bm3d.VBasic(p, sigma=sigma, radius=r, profile=_bm3d_profile)", + "bits_per_sample=16, sample_type=vs.INTEGER", + "format=_bm3d_fmt.id, dither_type=\"error_diffusion\")", + ] { + assert!(script.contains(want), "{name}: missing {want}"); + } + assert!(!script.contains("mvf.BM3D"), "{name}: the old mvsfunc call is back"); + assert!(!script.contains("core.knlm.KNLMeansCL("), "{name}: KNLMeans emitted too"); + assert!(!script.contains("{{NR_"), "{name}: unsubstituted NR placeholder"); + } + + // An unknown profile would be a hard error in the plugin. + let job = nr_job("test_169_bm3d_profile", NoiseReductionParameters { + method: NoiseReductionMethod::Bm3d, + bm3d_profile: "turbo".into(), + ..Default::default() + }); + assert!(script_text(&job).contains("_bm3d_profile = \"np\"")); +} + +/// KNLMeans runs on the device the probe found, with its clamped values. +#[test] +fn test_170_knlm_noise_reduction_uses_the_probed_device() { + create_output_dir(); + let job = nr_job("test_170_knlm", NoiseReductionParameters { + method: NoiseReductionMethod::KnlMeans, + knlm_h: 2.5, + knlm_d: 3, + knlm_a: 3, + knlm_s: 2, + knlm_chroma: false, + ..Default::default() + }); + let generator = ScriptGenerator::new().unwrap().with_knlm(Ok("auto".into())); + let (encode, preview) = both_scripts_with(&generator, &job).unwrap(); + for (name, script) in [("encode", &encode), ("preview", &preview)] { + assert!(script.contains( + "core.knlm.KNLMeansCL(clip, d=3, a=3, s=2, h=2.50, channels=\"Y\", device_type=\"auto\")"), + "{name}: luma call wrong"); + assert!(script.contains("if False and clip.format.color_family == vs.YUV:"), + "{name}: chroma pass not switched off"); + assert!(!script.contains("core.bm3d."), "{name}: BM3D emitted too"); + } +} + +/// Without a working OpenCL device a KNLMeans job is refused, before any +/// script exists, with the message the app shows. Other methods are unaffected. +#[test] +fn test_171_knlm_is_refused_without_a_device() { + create_output_dir(); + let message = "KNLMeans runs on the graphics card through OpenCL, and none was found."; + let generator = ScriptGenerator::new().unwrap().with_knlm(Err(message.into())); + let job = nr_job("test_171_knlm_refused", NoiseReductionParameters { + method: NoiseReductionMethod::KnlMeans, + ..Default::default() + }); + let err = both_scripts_with(&generator, &job).unwrap_err().to_string(); + assert!(err.contains(message), "refusal lost the reason: {err}"); + assert!( + !std::env::temp_dir().join(format!("{}.vpy", job.id)).exists(), + "a script was written for a job that cannot run" + ); + let bm3d = nr_job("test_171_bm3d_ok", NoiseReductionParameters { + method: NoiseReductionMethod::Bm3d, + ..Default::default() + }); + both_scripts_with(&generator, &bm3d).expect("BM3D needs no OpenCL"); +} + +fn upscale_files() -> vapourbox_worker::gpu_plugin::GpuPluginFiles { + vapourbox_worker::gpu_plugin::GpuPluginFiles::in_vapoursynth_dir( + vapourbox_worker::gpu_plugin::GpuPlugin::Upscale, + std::path::Path::new("/deps/macos-arm64/vapoursynth"), + "macos-arm64", + ) +} + +fn ai_upscale_job(name: &str, method: UpscaleMethod, factor: i32, noise: &str) -> VideoJob { + let mut job = create_base_job(name); + job.qtgmc_parameters.enabled = false; + job.processing_pipeline = Some(ProcessingPipeline { + deinterlace: QTGMCParameters { enabled: false, ..Default::default() }, + crop_resize: CropResizeParameters { + enabled: true, + use_integer_upscale: true, + upscale_method: method, + upscale_factor: factor, + cugan_noise: noise.into(), + ..Default::default() + }, + ..ProcessingPipeline::default() + }); + job +} + +/// Real-ESRGAN / Real-CUGAN: VSNCNN loaded explicitly from vapoursynth/ncnn/ +/// after the shared MoltenVK, the bundled model for the factor, the source's +/// matrix both ways, and none of the edge-directed loop — in both scripts. +#[test] +fn test_172_ai_upscale_emits_explicit_load_and_the_right_model() { + create_output_dir(); + let files = upscale_files(); + let py = |p: &std::path::Path| vapourbox_worker::gpu_plugin::py_str(p); + let generator = ScriptGenerator::new().unwrap().with_upscale(Ok(files.clone())); + for (method, model) in [ + (UpscaleMethod::RealEsrgan, "RealESRGANv2/RealESRGANv2-animevideo-xsx2.onnx"), + (UpscaleMethod::RealCugan, "cugan/up2x-latest-denoise3x.onnx"), + ] { + let job = ai_upscale_job("test_172_ai_upscale", method, 2, "denoise3x"); + let (encode, preview) = both_scripts_with(&generator, &job).unwrap(); + for (name, script) in [("encode", &encode), ("preview", &preview)] { + let ctx = format!("{method:?} {name}"); + assert!(script.contains(&format!("_ai_preload = {}", py(files.preload.as_deref().unwrap()))), + "{ctx}: MoltenVK not preloaded"); + assert!(script.contains(&format!("core.std.LoadPlugin({})", py(&files.plugin))), + "{ctx}: VSNCNN not loaded explicitly"); + assert!(script.contains(&format!("for _ai_model in [{}]:", py(&files.model_dir.join(model)))), + "{ctx}: wrong model"); + assert!(script.contains("core.ncnn.Model(_ai_rgb, network_path=_ai_model, fp16=True"), + "{ctx}: no ncnn.Model call"); + // 576 lines, untagged: BT.601 both ways, limited range. + assert!(script.contains(r#"matrix_in_s="170m", range_in_s="limited""#), "{ctx}: matrix in"); + assert!(script.contains(r#"matrix_s="170m", range_s="limited", dither_type="error_diffusion""#), + "{ctx}: matrix out"); + assert!(script.contains("clip = core.std.CopyFrameProps(clip, _ai_src)"), "{ctx}: props lost"); + assert!(!script.contains("_upscale_double_height"), "{ctx}: edge-directed loop emitted too"); + for prefix in ["{{AI_", "{{#UPSCALE_", "{{/UPSCALE_", "{{UPSCALE_"] { + assert!(!script.contains(prefix), "{ctx}: unsubstituted {prefix}"); + } + } + } +} + +/// 4x: Real-ESRGAN has a 4x model; Real-CUGAN's bundled models are 2x, so it +/// runs twice, denoising on the first pass only. +#[test] +fn test_173_ai_upscale_4x_models() { + create_output_dir(); + let files = upscale_files(); + let py = |m: &str| vapourbox_worker::gpu_plugin::py_str(&files.model_dir.join(m)); + let generator = ScriptGenerator::new().unwrap().with_upscale(Ok(files.clone())); + let esrgan = script_text_with(&generator, &ai_upscale_job("test_173_esrgan", UpscaleMethod::RealEsrgan, 4, "")); + assert!(esrgan.contains(&format!("for _ai_model in [{}]:", + py("RealESRGANv2/RealESRGANv2-animevideo-xsx4.onnx")))); + let cugan = script_text_with(&generator, &ai_upscale_job("test_173_cugan", UpscaleMethod::RealCugan, 4, "conservative")); + assert!(cugan.contains(&format!("for _ai_model in [{}, {}]:", + py("cugan/up2x-latest-conservative.onnx"), py("cugan/up2x-latest-no-denoise.onnx")))); +} + +/// Without a working Vulkan device an AI upscale job is refused, before any +/// script exists, with the probe's message. The other upscalers are unaffected. +#[test] +fn test_174_ai_upscale_is_refused_where_it_cannot_run() { + create_output_dir(); + let message = "AI upscaling needs a graphics card with Vulkan support, and none was found."; + let generator = ScriptGenerator::new().unwrap().with_upscale(Err(message.into())); + let job = ai_upscale_job("test_174_refused", UpscaleMethod::RealCugan, 2, "conservative"); + let err = both_scripts_with(&generator, &job).unwrap_err().to_string(); + assert!(err.contains(message), "refusal lost the reason: {err}"); + assert!(!std::env::temp_dir().join(format!("{}.vpy", job.id)).exists()); + let nnedi = ai_upscale_job("test_174_nnedi3", UpscaleMethod::Nnedi3Rpow2, 2, ""); + let (encode, _) = both_scripts_with(&generator, &nnedi).expect("NNEDI3 needs no GPU"); + assert!(!encode.contains("core.ncnn."), "a non-AI upscale emitted VSNCNN"); +} + +fn script_text_with(generator: &ScriptGenerator, job: &VideoJob) -> String { + std::fs::read_to_string(generator.generate(job).expect("generate")).expect("read script") +} + +fn descale_job(name: &str, ds: DescaleParameters) -> VideoJob { + let mut job = create_base_job(name); + job.qtgmc_parameters.enabled = false; + job.processing_pipeline = Some(ProcessingPipeline { + deinterlace: QTGMCParameters { enabled: false, ..Default::default() }, + descale: DescaleParameters { enabled: true, ..ds }, + ..ProcessingPipeline::default() + }); + job +} + +/// Every kernel's inverse is paired with the same kernel's forward resize (the +/// protection mask compares against it), in both scripts. +#[test] +fn test_175_descale_emits_the_kernel_pair_and_the_rescale() { + create_output_dir(); + for (kernel, inverse, forward) in [ + (DescaleKernel::Bilinear, "core.descale.Debilinear(c, w, h, border_handling=0)", "core.resize.Bilinear(c, w, h)"), + (DescaleKernel::Bicubic, "core.descale.Debicubic(c, w, h, b=0.25, c=0.4, border_handling=0)", + "core.resize.Bicubic(c, w, h, filter_param_a=0.25, filter_param_b=0.4)"), + (DescaleKernel::Lanczos, "core.descale.Delanczos(c, w, h, taps=4, border_handling=0)", "core.resize.Lanczos(c, w, h, filter_param_a=4)"), + (DescaleKernel::Spline36, "core.descale.Despline36(c, w, h, border_handling=0)", "core.resize.Spline36(c, w, h)"), + ] { + let job = descale_job("test_175_descale", DescaleParameters { + native_height: 810, + kernel, + bicubic_b: 0.25, + bicubic_c: 0.4, + lanczos_taps: 4, + ..Default::default() + }); + let (encode, preview) = both_scripts_with(&ScriptGenerator::new().unwrap(), &job).unwrap(); + for (name, script) in [("encode", &encode), ("preview", &preview)] { + let ctx = format!("{kernel:?} {name}"); + assert!(script.contains(&format!(" return {inverse}")), "{ctx}: inverse"); + assert!(script.contains(&format!(" return {forward}")), "{ctx}: forward"); + assert!(script.contains("_ds_h = 810"), "{ctx}: height"); + assert!(script.contains("_ds_w = 0 or int(round("), "{ctx}: width from shape"); + assert!(script.contains("_nnedi3(_ds_up, field=1, dh=True"), "{ctx}: no NNEDI3 re-upscale"); + assert!(script.contains("core.std.MaskedMerge(_ds_up, _ds_y, _ds_mask)"), "{ctx}: no protection"); + assert!(script.contains("core.std.Binarize(_ds_error, 4.0 / 255.0)"), "{ctx}: threshold"); + assert!(script.contains("core.std.ShufflePlanes([_ds_luma, _ds_src], [0, 1, 2]"), "{ctx}: chroma not kept"); + assert!(!script.contains("{{DESCALE") && !script.contains("{{#DESCALE") && !script.contains("{{/DESCALE"), + "{ctx}: unsubstituted placeholder"); + } + } +} + +/// Protection off leaves no mask; a set native width is used as given. +#[test] +fn test_176_descale_options() { + create_output_dir(); + let job = descale_job("test_176_descale", DescaleParameters { + protect_detail: false, + native_width: 1500, + border: DescaleBorder::Extend, + ..Default::default() + }); + let script = script_text(&job); + assert!(!script.contains("_ds_mask"), "protection emitted when off"); + assert!(script.contains("_ds_w = 1500 or")); + assert!(script.contains("border_handling=2)")); +} + +/// Descale runs before noise reduction (which would change the upscale it +/// inverts), and nothing of it is emitted when the pass is off. +#[test] +fn test_177_descale_order_and_off() { + create_output_dir(); + let mut job = descale_job("test_177_descale", DescaleParameters::default()); + if let Some(p) = job.processing_pipeline.as_mut() { + p.noise_reduction = NoiseReductionParameters { enabled: true, ..Default::default() }; + } + let script = script_text(&job); + let ds = script.find("_ds_native = _ds_descale(").expect("descale call"); + let nr = script.find("haf.SMDegrain(").expect("noise reduction call"); + assert!(ds < nr, "descale must run before noise reduction"); + + let mut off = create_base_job("test_177_off"); + off.qtgmc_parameters.enabled = false; + off.processing_pipeline = Some(ProcessingPipeline { + deinterlace: QTGMCParameters { enabled: false, ..Default::default() }, + ..ProcessingPipeline::default() + }); + assert!(!script_text(&off).contains("_ds_"), "descale emitted with the pass off"); +} From 3c625720fb518a8d28f3da7397d62351221ebc3d Mon Sep 17 00:00:00 2001 From: Stuart Cameron Date: Sun, 25 Oct 2026 12:24:25 +1100 Subject: [PATCH 02/15] =?UTF-8?q?feat:=20AnimeJaNai=20upscaling=20as=20an?= =?UTF-8?q?=20add-on;=20one=20add-on=20system=20with=20Settings=20?= =?UTF-8?q?=E2=86=92=20Add-ons?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Crop & Resize: AnimeJaNai (the-database's Real-ESRGAN Compact models, CC BY-NC-SA 4.0) on VSNCNN. Its SD model is used up to 576 lines and the chosen HD model (Compact / UltraCompact / SuperUltraCompact, optionally Sharp) above, decided in the script per doubling. Flat colour stays within 0.5 levels. - The models are never in the deps bundle. They're downloaded from upstream's 3.0.0 release, with the zip and each model pinned by SHA-256, and the non-commercial terms are shown at download time and while the method is chosen. The worker refuses the job if the models are missing. - AddonService replaces the Whisper-only manager and dialogs: one catalogue, requirements installed first, dependants removed together. Settings → Add-ons lists every add-on with its size, licence and source, to install or remove, and AddonPrompt is the shared on-demand prompt. - Schema: ui.optionLabels, so dropdowns show "Real-ESRGAN" instead of "Realesrgan". - Credits: the Whisper models (MIT, OpenAI) and AnimeJaNai are now in NOTICES, About and README. The RemoveGrain About tile now names its MIT component, and the VSNCNN tile names LLVM OpenMP. - Tests: About and NOTICES licences must match; each add-on's credits must agree across NOTICES, About, README and Settings; the GPU tests install the real AnimeJaNai download. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 67 ++- README.md | 9 +- app/assets/animejanai-addon.json | 20 + app/assets/filters/core/crop_resize.json | 66 ++- app/lib/models/crop_resize_parameters.dart | 16 + app/lib/models/filter_schema.dart | 6 + app/lib/models/gpu_plugin_status.dart | 9 +- app/lib/models/parameter_converter.dart | 4 + app/lib/services/addon_service.dart | 374 +++++++++++++++ app/lib/services/whisper_addon_manager.dart | 425 +++++++----------- app/lib/views/about_dialog.dart | 21 +- app/lib/views/addon_dialogs.dart | 224 +++++++++ app/lib/views/pass_list/pass_list_panel.dart | 22 +- .../pass_settings/pass_settings_inline.dart | 109 ++++- .../views/settings/addons_settings_tab.dart | 163 +++++++ app/lib/views/settings/settings_dialog.dart | 5 +- .../settings/widgets/parameter_widgets.dart | 2 +- app/lib/views/whisper_download_dialog.dart | 239 ---------- app/pubspec.yaml | 1 + app/test/addon_service_test.dart | 297 ++++++++++++ app/test/addons_ui_widget_test.dart | 120 +++++ app/test/attribution_test.dart | 56 +++ app/test/gpu_plugin_status_test.dart | 34 +- app/test/integration_ai_upscale_test.dart | 71 ++- app/test/integration_gpu_plugins_test.dart | 17 +- app/test/support/worker_harness.dart | 31 ++ docs/ENGINEERING_NOTES.md | 39 ++ docs/FILTER_SCHEMA.md | 1 + licenses/NOTICES.txt | 23 +- worker/src/animejanai.rs | 125 ++++++ worker/src/dependency_locator.rs | 7 + worker/src/lib.rs | 1 + worker/src/main.rs | 7 +- worker/src/models/crop_resize_parameters.rs | 100 ++++- worker/src/models/processing_pipeline.rs | 6 + worker/src/pipeline_executor.rs | 3 +- worker/src/script_generator.rs | 37 +- worker/templates/pipeline_template.vpy | 4 + worker/templates/preview_template.vpy | 4 + worker/tests/filter_integration_test.rs | 52 +++ 40 files changed, 2226 insertions(+), 591 deletions(-) create mode 100644 app/assets/animejanai-addon.json create mode 100644 app/lib/services/addon_service.dart create mode 100644 app/lib/views/addon_dialogs.dart create mode 100644 app/lib/views/settings/addons_settings_tab.dart delete mode 100644 app/lib/views/whisper_download_dialog.dart create mode 100644 app/test/addon_service_test.dart create mode 100644 app/test/addons_ui_widget_test.dart create mode 100644 worker/src/animejanai.rs diff --git a/CLAUDE.md b/CLAUDE.md index c9c4a4d8..5ec014b7 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -103,6 +103,7 @@ VapourBox/ | `worker/src/cpu.rs` | The x86 deps CPU tier (`v3`/`v2`) — the single decision; see "CPU tiers" | | `worker/src/source_decode.rs` | Decoder ffmpeg args for encode + preview: full stored frame, size-guarded (see "Source Frame Size") | | `worker/src/gpu_plugin.rs` | The ncnn-on-Vulkan plugins (RIFE, AI upscaling): file layout, the `--probe-rife` / `--probe-upscale` probe, and the user-facing reason/remedy when they can't run (see "GPU Plugins") | +| `worker/src/animejanai.rs` | AnimeJaNai's add-on models: where they live, which one a job runs, the refusal when they're missing (see "Add-ons") | | `worker/src/native_res.rs` | `--detect-native`: the resolution and kernel a source was upscaled from, for the Descale pass (see "Descale") | | `worker/templates/pipeline_template.vpy` | VapourSynth script template | | `worker/tests/filter_integration_test.rs` | Filter integration tests | @@ -119,6 +120,7 @@ VapourBox/ | `app/lib/services/temp_directory_service.dart` | Configurable scratch-file directory (see "Temporary Files Directory") | | `app/lib/services/overwrite_behavior_service.dart` | Default action for existing output files (see "Existing Output Files") | | `app/lib/models/native_res_report.dart` | What `--detect-native` found, as the Descale panel's Detect shows and applies it | +| `app/lib/services/addon_service.dart` | Every optional third-party download (Whisper, AnimeJaNai): catalogue, install with requirements, remove with dependants (see "Add-ons") | | `app/lib/models/gpu_plugin_status.dart` | The worker's GPU-plugin probe results, and the panels' notice decision | | `app/assets/filters/core/*.json` | Built-in filter schema definitions | @@ -1057,8 +1059,8 @@ asserted in `shutdown_when_done_test.dart`: Two features run a neural network through ncnn on Vulkan: **RIFE** 4.26 (`rife.RIFE`), the Frame Rate pass's `FrameRateMethod::Rife`, and **AI upscaling** (vs-mlrt's VSNCNN, `ncnn.Model`, with Real-ESRGAN's anime video -model or Real-CUGAN), Crop & Resize's `UpscaleMethod::RealEsrgan` / -`RealCugan`. `worker/src/gpu_plugin.rs` (`GpuPlugin::{Rife, Upscale}`) holds +model, Real-CUGAN or AnimeJaNai), Crop & Resize's `UpscaleMethod::RealEsrgan` / +`RealCugan` / `AnimeJanai`. `worker/src/gpu_plugin.rs` (`GpuPlugin::{Rife, Upscale}`) holds everything they share, and the rules all follow from one fact: **without a usable Vulkan implementation ncnn can segfault** (RIFE does, in `ncnn::create_gpu_instance`; VSNCNN raises "get_gpu_device failed") rather @@ -1108,6 +1110,15 @@ than fail cleanly. matrix/range, odd sizes edge-padded, frame props copied back, fp16 (≤0.3/255 from fp32, ~3x faster). Real-ESRGAN measurably tints flat colour (up to ~11 levels at 4x, in fp32 too); that is the model, not the pipeline. +- **AnimeJaNai** (`UpscaleMethod::AnimeJanai`) runs on the same plugin and + block, but its models are an **add-on**, never in the deps bundle — they are + CC BY-NC-SA 4.0, non-commercial only, and the bundle must stay freely + redistributable (see "Add-ons"). Each doubling is an `(sd, hd)` pair the + script picks by height (`_ai_rgb.height <= 576` → the SD model), so the size + and Sharp settings only reach HD pictures, and a 4x DVD runs SD then HD. + `ensure_methods_ready` also refuses it when `animejanai::runtime` finds the + models missing (`MISSING_MESSAGE`). It keeps flat colour within 0.5 levels + (Real-ESRGAN: up to 11). - **VSNCNN is built from source on macOS and Linux** (only Windows has a published build): static protobuf v3.21.12 and onnx at vs-mlrt's pinned commit, Tencent's prebuilt ncnn 20260526, VapourSynth's API 3 headers from @@ -1164,6 +1175,37 @@ keeps chroma and the source size. Rules: `integration_descale_test.dart` (heavy) checks detection on known masters, protection (grid exact vs. destroyed without the mask) and preview = encode. +### Add-ons: optional third-party downloads + +Anything VapourBox can use but does not ship — because of its size (Whisper) +or its licence (AnimeJaNai) — is an **add-on**: downloaded from its upstream +source into the add-ons directory (`DependencyLocator::addons_dir`, mirrored +by `AddonService.addonsDir`), only when the user asks. Rules: + +- **One service.** `AddonService` holds the catalogue (`AddonInfo`: size, + licence, credit, source, `usageNote` for a restriction, `requires`) and one + `AddonInstaller` per add-on. **Settings → Add-ons** (`AddonsSettingsTab`) + lists, installs and removes; a feature that needs one calls + `AddonPrompt.ensure(context, ids, reason:)`, which states the source, + licence and size before downloading. Never add a feature-specific download + dialog. Installing installs requirements first (a Whisper model needs the + engine); removing removes what requires it, after saying so. +- **A licence restriction is shown, not buried**: `usageNote` is bold in the + prompt, the Settings row and (for AnimeJaNai) the panel notice whenever the + method is chosen. Such an add-on also gets a NOTICES entry and About tile + that say it is a separate download. +- **Pinned and verified**: `assets/animejanai-addon.json` pins upstream's + release asset by SHA-256 *and* each model by its own hash; extraction + (`AnimeJanaiInstaller.extractModels`) keeps only the listed files, stages + beside the target and renames into place, so a bad download installs + nothing. Upstream re-uploaded that asset once (2024-07), which is why the + pin matters: a changed asset fails closed. +- **The worker checks presence, the app checks hashes.** `animejanai::MODELS` + and the manifest list the same files (asserted on both sides). +- **Tested against the real download**: `WorkerHarness.ensureAnimeJanai()` + installs from the pinned URL with the app's own extraction, so CI fails if + the link or hashes stop holding. + ### BM3D and KNLMeans Noise Reduction Two advanced-only Noise Reduction methods (`bm3d`, `knlmeans`, app 1.5.0; @@ -1472,12 +1514,21 @@ committed under `Tests/TestResources/`. ## Attribution — never write a name you have not read upstream -Third-party credit lives in **three** places that must agree: +Third-party credit lives in **four** places that must agree: `licenses/NOTICES.txt` (shipped in every package), the About dialog -(`app/lib/views/about_dialog.dart`), and the README's Acknowledgments. -`app/test/attribution_test.dart` lints the first two against -`Scripts/deps-expected-plugins.json`, so **adding a plugin without crediting it -fails the build**. +(`app/lib/views/about_dialog.dart`), the README's Acknowledgments, and — for an +add-on — its `AddonInfo` (licence, credit, `noticeName`), which Settings → +Add-ons and the download prompt display. Guarded by: + +- `app/test/attribution_test.dart`: NOTICES and About against + `Scripts/deps-expected-plugins.json`, so **adding a plugin without crediting + it fails the build**; and every About tile states the same licence(s) as its + NOTICES entry (compared as licence families, so a dropped component licence + fails — RemoveGrain's tile once omitted its MIT part). +- `app/test/addon_service_test.dart` ("credits agree everywhere"): each add-on's + NOTICES entry has its licence and credit and says it is a download, it has an + About tile with the same licence, the README credits it, and a + non-commercial licence carries a `usageNote`. **Every copyright line must come from the upstream LICENSE file or a source header, fetched at the time of writing — never from a GitHub handle expanded @@ -2084,4 +2135,4 @@ Full write-ups (root causes, measurements) for each entry are in | 1.11.0 | 2026-09 | **Issue #92**: every x86 bundle ships in two **CPU tiers** — `v3` (x86-64-v3; the plain asset names) and `v2` (`…-x64-v2`, anything older), chosen by the app from `vapourbox-worker --probe-cpu`. Each bundle carries one autoloaded zsmooth (replacing 1.10.0's two-builds-loaded-by-path). macOS v2 builds MVTools v24 from source without its AVX2 files, whose static initializers SIGILLed inside `dlopen` on pre-AVX Macs. v2 bundles are gated (SDE on Linux/Windows, static check on macOS) before publishing | | 1.12.0 | 2026-10-04 | **Issue #101**: akarin 1.4.1 → **1.5.0** on every platform. The 1.4.1 Linux wheel put JIT code on the `brk` heap, where SELinux denies `PROT_EXEC`, so every job segfaulted on Fedora/RHEL from deps 1.8.0 on. `download-deps-linux.sh` now fails if `libakarin.so` lacks the mmap allocator | | 1.13.0 | 2026-10 | One new plugin; every other binary is built from the same sources as 1.12.0. Adds **vszip** 22.1.0 (MIT) for `vszip.CLAHE`, Color Correction's local contrast (issue #90). Baseline build only, flat in the plugin directory on every platform (the wheels' AVX2/AVX-512 variants are left out); macOS x64 builds the sdist with Zig 0.16.0 for the 12.0 floor, with header padding so it can be signed. The v2 CPU tier is no longer a second full bundle: each x64 platform publishes its v3 bundle plus `…-x64-v2-delta.zip`, the files in `_tierFiles` (zsmooth; on macOS also MVTools), extracted over it (`"tierFormat": "delta"`). Every x64 machine downloads the same bundle; only an older CPU fetches the few MB more. This is also what lets the Linux AppImage carry its deps for both tiers without doubling in size | -| 1.14.0 | unreleased | Adds **RIFE** 4.26 (`r9_mod_v33`, MIT) + its model in `vapoursynth/rife/`, and **VSNCNN** (vs-mlrt v15.16, GPL-3.0) + Real-ESRGAN / Real-CUGAN models in `vapoursynth/ncnn/`, on macos-arm64, windows-x64 and linux-x64, with **MoltenVK** 1.4.2 (Apache-2.0) in `vapoursynth/moltenvk/` on macOS. Not autoloaded; see "GPU Plugins". **BM3D** r12 (MIT) on every platform; it had been macOS arm64 only (see "BM3D and KNLMeans Noise Reduction"). **descale** r12 (MIT) on every platform, for the Descale pass | +| 1.14.0 | unreleased | Adds **RIFE** 4.26 (`r9_mod_v33`, MIT) + its model in `vapoursynth/rife/`, and **VSNCNN** (vs-mlrt v15.16, GPL-3.0) + Real-ESRGAN / Real-CUGAN models in `vapoursynth/ncnn/`, on macos-arm64, windows-x64 and linux-x64, with **MoltenVK** 1.4.2 (Apache-2.0) in `vapoursynth/moltenvk/` on macOS. Not autoloaded; see "GPU Plugins". **BM3D** r12 (MIT) on every platform; it had been macOS arm64 only (see "BM3D and KNLMeans Noise Reduction"). **descale** r12 (MIT) on every platform, for the Descale pass. (AnimeJaNai's models, used by the same VSNCNN, are an app add-on, not in the bundle) | diff --git a/README.md b/README.md index 6a4ba3de..8aea28db 100644 --- a/README.md +++ b/README.md @@ -85,7 +85,9 @@ GPU-accelerated deinterlacing (NNEDI3CL) needs your GPU's OpenCL driver installe -**RIFE frame interpolation** (in the Frame Rate pass) and **AI upscaling** with Real-ESRGAN or Real-CUGAN (in Crop & Resize) run on the GPU through Vulkan. They work on any Apple Silicon Mac, and on Windows and Linux x64 PCs with a current graphics driver; RIFE on Windows also needs the Microsoft Visual C++ Redistributable. They aren't available on Intel Macs or Linux on ARM yet. When you choose one, VapourBox checks that it can run and, if it can't, says what is missing and how to fix it. +**RIFE frame interpolation** (in the Frame Rate pass) and **AI upscaling** with Real-ESRGAN, Real-CUGAN or AnimeJaNai (in Crop & Resize) run on the GPU through Vulkan. They work on any Apple Silicon Mac, and on Windows and Linux x64 PCs with a current graphics driver; RIFE on Windows also needs the Microsoft Visual C++ Redistributable. They aren't available on Intel Macs or Linux on ARM yet. When you choose one, VapourBox checks that it can run and, if it can't, says what is missing and how to fix it. + +**Add-ons.** Some features use third-party downloads that aren't part of VapourBox: the Whisper speech-recognition engine and its language models for Subtitles, and the AnimeJaNai models for AI upscaling. VapourBox offers to download one the first time you choose a feature that needs it, saying where it comes from and under what licence. **Settings → Add-ons** lists them all, to install or remove at any time. The AnimeJaNai models are free for **non-commercial use only** (CC BY-NC-SA 4.0); if you use VapourBox for paid work, use Real-ESRGAN or Real-CUGAN instead. **Older Intel and AMD processors** (before about 2013 — no AVX2) are supported on all three platforms. On first launch VapourBox checks the processor and installs the matching set of processing components; a machine moved to a different CPU gets the right set on its next launch. The older-CPU set runs some filters more slowly, never differently. Every build of that set is checked for pre-AVX processors (e.g. a 2010 Mac Pro) on macOS and Linux; on Windows it is checked for processors with AVX but no AVX2, and older ones are expected to work but can't be tested automatically. @@ -124,7 +126,7 @@ Twenty-two filters, each switchable independently. Most sources need none or a f | 15 | **Color Correction** | Brightness, contrast, saturation, hue, levels, white balance (warm/cool, green/magenta), lifting detail out of the shadows of underexposed footage, and lifting local contrast across a dim or flat picture. Levels and white balance can each be measured automatically or set by hand. | | 16 | **Stabilize** | Shake and weave — telecine wobble, jittery film scans, handheld footage. Runs last before cropping, so a small crop removes the edges it exposes. | | 17 | **Rotate / Flip** | Footage shot sideways, mirrored captures, scans that came off the scanner the wrong way round. | -| 18 | **Crop & Resize** | Trimming overscan, scaling, and upscaling — edge-directed, or with Real-ESRGAN or Real-CUGAN, AI upscalers trained on anime that run on the GPU — plus bars in a colour of your choice to bring a cropped picture back to an exact frame size (720×576 for PAL DVD, 720×480 for NTSC) without rescaling it. | +| 18 | **Crop & Resize** | Trimming overscan, scaling, and upscaling — edge-directed, or with Real-ESRGAN, Real-CUGAN or AnimeJaNai (an optional, non-commercial download), AI upscalers trained on anime that run on the GPU — plus bars in a colour of your choice to bring a cropped picture back to an exact frame size (720×576 for PAL DVD, 720×480 for NTSC) without rescaling it. | | 19 | **Sharpen** | Soft sources needing edge and fine detail recovery. aWarpSharp2 sharpens by warping edges instead of raising contrast, so it adds no halos. Runs after the resize, on the picture that is actually delivered, and before any added grain. | | 20 | **Film Grain** | Grain added back after denoising, so the picture is not left plastic — and to hide banding in skies and fades. | | 21 | **Frame Rate** | Converting between PAL and NTSC rates, for a tape that was already converted once and now plays at the wrong speed. New frames come from motion interpolation, from RIFE (an AI interpolator that runs on the GPU), or from plain frame repetition. | @@ -304,7 +306,8 @@ Stuart Cameron — [stuart-cameron.com](https://stuart-cameron.com) - **vszip** by Julek — CLAHE, the local-contrast lift in Color Correction - **RIFE** by hzwer (with Megvii) — the frame interpolation network, run through HolyWu's VapourSynth plugin (built on nihui's rife-ncnn-vulkan and Tencent's ncnn), with **MoltenVK** for Vulkan on macOS - **Real-ESRGAN** by Xintao Wang and **Real-CUGAN** by bilibili — the AI upscaling networks, run through AmusementClub's **vs-mlrt** (VSNCNN) -- **whisper.cpp** by the ggml authors — speech recognition for subtitle generation +- **AnimeJaNai** by the-database — the anime upscaling models offered as an optional download (CC BY-NC-SA 4.0, non-commercial use only) +- **whisper.cpp** by the ggml authors and OpenAI's **Whisper** models — speech recognition for subtitle generation, an optional download - **libdvdread** by VideoLAN — DVD reading and navigation - **FFmpeg** project — video encoding - **Hybrid** by Selur — inspiration for this project diff --git a/app/assets/animejanai-addon.json b/app/assets/animejanai-addon.json new file mode 100644 index 00000000..79542335 --- /dev/null +++ b/app/assets/animejanai-addon.json @@ -0,0 +1,20 @@ +{ + "version": "3.0.0", + "licence": "CC BY-NC-SA 4.0", + "licenceUrl": "https://creativecommons.org/licenses/by-nc-sa/4.0/", + "homepage": "https://github.com/the-database/mpv-upscale-2x_animejanai", + "archive": { + "url": "https://github.com/the-database/mpv-upscale-2x_animejanai/releases/download/3.0.0/2x_AnimeJaNai_HD_V3_ModelsOnly.zip", + "size": 24674026, + "sha256": "50eefb2b657a19934f33ca002ab8f701b5c960566bd54b23f658bdeb19d7788f" + }, + "models": { + "2x_AnimeJaNai_HD_V3_Compact.onnx": "053178394c0962bab3c2a6d3262d6d96a1a944eee2fcad31cad7ec22034a8898", + "2x_AnimeJaNai_HD_V3_UltraCompact.onnx": "1d270c7f7d9a09bb3af59cca4f79e1554c429c804255dccef0963d263f843c22", + "2x_AnimeJaNai_HD_V3_SuperUltraCompact.onnx": "f6bbfbed502b8d300ee9d678d2c277d4b56235c36d053c3a1a41be060c54c0e6", + "2x_AnimeJaNai_HD_V3Sharp1_Compact.onnx": "15258aa92f53ed55c6cf488b30f527a8087ebca929535a4dd33b410f5fa87de6", + "2x_AnimeJaNai_HD_V3Sharp1_UltraCompact.onnx": "a01e4f262f68708639dfd7fabdb971c2d467d616a3cb9293692fc9610b4e0ab4", + "2x_AnimeJaNai_HD_V3Sharp1_SuperUltraCompact.onnx": "bcf1e3320eb7181f7f3e25901d5f7a681b1ab8a212df4be613ca7da109d960f7", + "2x_AnimeJaNai_SD_V1beta34_Compact.onnx": "d361a6eff99b9dbfe138670514bd84191f8e0b288085ab397d3ed8d3e03e52dc" + } +} diff --git a/app/assets/filters/core/crop_resize.json b/app/assets/filters/core/crop_resize.json index 2473223c..3906c393 100644 --- a/app/assets/filters/core/crop_resize.json +++ b/app/assets/filters/core/crop_resize.json @@ -248,12 +248,21 @@ "eedi3Rpow2", "spline36", "realEsrgan", - "realCugan" + "realCugan", + "animeJanai" ], "optional": true, "ui": { "label": "Upscale Method", - "description": "Edge-directed doubling. NNEDI3 is the usual choice; EEDI3 is slower and follows near-horizontal edges better (it uses NNEDI3 to guide its search); Spline36 is plain resampling with no edge direction. Real-ESRGAN and Real-CUGAN are neural upscalers trained on anime: they redraw clean line art rather than interpolating it, and run on the graphics card (Vulkan). Real-ESRGAN is sharper and faster; Real-CUGAN is gentler and can remove noise as it goes. Use them after deinterlacing: on a combed picture they sharpen the combing too", + "optionLabels": { + "nnedi3Rpow2": "NNEDI3", + "eedi3Rpow2": "EEDI3", + "spline36": "Spline36", + "realEsrgan": "Real-ESRGAN", + "realCugan": "Real-CUGAN", + "animeJanai": "AnimeJaNai (download, non-commercial)" + }, + "description": "Edge-directed doubling. NNEDI3 is the usual choice; EEDI3 is slower and follows near-horizontal edges better (it uses NNEDI3 to guide its search); Spline36 is plain resampling with no edge direction. Real-ESRGAN and Real-CUGAN are neural upscalers trained on anime: they redraw clean line art rather than interpolating it, and run on the graphics card (Vulkan). Real-ESRGAN is sharper and faster; Real-CUGAN is gentler and can remove noise as it goes. AnimeJaNai, trained on HD anime, keeps colour exactly and comes in three speeds; it is a separate download, free for non-commercial use only. Use them after deinterlacing: on a combed picture they sharpen the combing too", "widget": "dropdown", "visibleWhen": { "useIntegerUpscale": true @@ -298,6 +307,47 @@ } } }, + "animejanaiModel": { + "type": "enum", + "default": "compact", + "options": [ + "compact", + "ultraCompact", + "superUltraCompact" + ], + "sinceAppVersion": "1.5.0", + "ui": { + "label": "AnimeJaNai Model", + "optionLabels": { + "compact": "Compact (best)", + "ultraCompact": "UltraCompact (about 2x faster)", + "superUltraCompact": "SuperUltraCompact (about 4x faster)" + }, + "description": "The HD model's size: smaller is faster and a little less detailed. A picture 576 lines or shorter (DVD) always uses AnimeJaNai's SD model, which comes in one size; at 4x a DVD runs the SD model and then this one", + "widget": "dropdown", + "visibleWhen": { + "useIntegerUpscale": true, + "upscaleMethod": [ + "animeJanai" + ] + } + } + }, + "animejanaiSharp": { + "type": "boolean", + "default": false, + "sinceAppVersion": "1.5.0", + "ui": { + "label": "AnimeJaNai Sharp", + "description": "Use the Sharp variant of the HD model, which draws crisper lines. The SD model has no Sharp variant", + "visibleWhen": { + "useIntegerUpscale": true, + "upscaleMethod": [ + "animeJanai" + ] + } + } + }, "bicubicB": { "type": "number", "default": 0.0, @@ -869,7 +919,9 @@ "useIntegerUpscale", "upscaleMethod", "upscaleFactor", - "cuganNoise" + "cuganNoise", + "animejanaiModel", + "animejanaiSharp" ], "expanded": false }, @@ -951,6 +1003,14 @@ "upscaleMethod": "realCugan" } }, + { + "function": "core.ncnn.Model", + "role": "AnimeJaNai (add-on), on the GPU (vs-mlrt's VSNCNN)", + "activeWhen": { + "useIntegerUpscale": true, + "upscaleMethod": "animeJanai" + } + }, { "function": "core.std.AddBorders", "role": "Pad to Fill's letterbox/pillarbox bars, in the border colour", diff --git a/app/lib/models/crop_resize_parameters.dart b/app/lib/models/crop_resize_parameters.dart index 65d706d1..e4cfb250 100644 --- a/app/lib/models/crop_resize_parameters.dart +++ b/app/lib/models/crop_resize_parameters.dart @@ -38,6 +38,10 @@ enum UpscaleMethod { /// Real-CUGAN, on the GPU (needs Vulkan). @JsonValue('realCugan') realCugan, + /// AnimeJaNai, on the GPU (needs Vulkan). Its models are an add-on, + /// non-commercial use only, downloaded when it is chosen. + @JsonValue('animeJanai') + animeJanai, } /// What to do with the pixel aspect ratio (SAR) on output. @@ -186,6 +190,12 @@ class CropResizeParameters { /// Real-CUGAN's denoise strength: none, conservative or denoise3x. final String cuganNoise; + /// AnimeJaNai's HD model size: compact, ultraCompact or superUltraCompact. + final String animejanaiModel; + + /// AnimeJaNai's Sharp variant of the HD model. + final bool animejanaiSharp; + // --- nnedi3 controls for the edge-directed upscale paths --- // With EEDI3 selected these shape the nnedi3 sclip that guides it. @@ -255,6 +265,8 @@ class CropResizeParameters { this.upscaleMethod = UpscaleMethod.nnedi3Rpow2, this.upscaleFactor = 2, this.cuganNoise = 'conservative', + this.animejanaiModel = 'compact', + this.animejanaiSharp = false, this.upscaleNsize, this.upscaleNeurons, this.upscaleQual, @@ -351,6 +363,8 @@ class CropResizeParameters { UpscaleMethod? upscaleMethod, int? upscaleFactor, String? cuganNoise, + String? animejanaiModel, + bool? animejanaiSharp, int? upscaleNsize, int? upscaleNeurons, int? upscaleQual, @@ -391,6 +405,8 @@ class CropResizeParameters { upscaleMethod: upscaleMethod ?? this.upscaleMethod, upscaleFactor: upscaleFactor ?? this.upscaleFactor, cuganNoise: cuganNoise ?? this.cuganNoise, + animejanaiModel: animejanaiModel ?? this.animejanaiModel, + animejanaiSharp: animejanaiSharp ?? this.animejanaiSharp, upscaleNsize: upscaleNsize ?? this.upscaleNsize, upscaleNeurons: upscaleNeurons ?? this.upscaleNeurons, upscaleQual: upscaleQual ?? this.upscaleQual, diff --git a/app/lib/models/filter_schema.dart b/app/lib/models/filter_schema.dart index bc05cb99..7089ddee 100644 --- a/app/lib/models/filter_schema.dart +++ b/app/lib/models/filter_schema.dart @@ -74,6 +74,11 @@ class ParameterUiConfig { /// Example: {"true": "Top Field First", "false": "Bottom Field First"} final Map? booleanLabels; + /// Display names for an enum's options, keyed by option value. An option + /// without one is shown title-cased (`spline36` → `Spline36`), which mangles + /// a camel-case name like `realEsrgan`. + final Map? optionLabels; + /// Extensions the file picker offers, without dots — `["srt", "ass"]`. /// Only meaningful for [WidgetType.filepicker]; null means any file. final List? fileExtensions; @@ -86,6 +91,7 @@ class ParameterUiConfig { this.hidden, this.visibleWhen, this.booleanLabels, + this.optionLabels, this.fileExtensions, }); diff --git a/app/lib/models/gpu_plugin_status.dart b/app/lib/models/gpu_plugin_status.dart index 0bbea1c1..0e7e8a90 100644 --- a/app/lib/models/gpu_plugin_status.dart +++ b/app/lib/models/gpu_plugin_status.dart @@ -81,6 +81,13 @@ GpuPluginNotice gpuPluginNoticeFor({ return status.available ? GpuPluginNotice.none : GpuPluginNotice.unavailable; } +/// Whether a panel's current values choose the AnimeJaNai upscaler, whose +/// models are an add-on (on top of the GPU plugin [gpuPluginSelectedBy] finds). +bool animeJanaiSelectedBy(String filterId, Map values) => + filterId == 'crop_resize' && + values['useIntegerUpscale'] == true && + values['upscaleMethod'] == 'animeJanai'; + /// Which GPU feature, if any, a panel's current values select. GpuPlugin? gpuPluginSelectedBy(String filterId, Map values) { switch (filterId) { @@ -89,7 +96,7 @@ GpuPlugin? gpuPluginSelectedBy(String filterId, Map values) { case 'crop_resize': final method = values['upscaleMethod']; return values['useIntegerUpscale'] == true && - (method == 'realEsrgan' || method == 'realCugan') + (method == 'realEsrgan' || method == 'realCugan' || method == 'animeJanai') ? GpuPlugin.upscale : null; } diff --git a/app/lib/models/parameter_converter.dart b/app/lib/models/parameter_converter.dart index 6d416651..6c11205b 100644 --- a/app/lib/models/parameter_converter.dart +++ b/app/lib/models/parameter_converter.dart @@ -916,6 +916,8 @@ class ParameterConverter { 'upscaleMethod': params.upscaleMethod.name, 'upscaleFactor': params.upscaleFactor, 'cuganNoise': params.cuganNoise, + 'animejanaiModel': params.animejanaiModel, + 'animejanaiSharp': params.animejanaiSharp, }; // Kernel tuning and the nnedi3/EEDI3 controls start unticked, so the // plugin's own default applies until the user opts in. Fallbacks are those @@ -1545,6 +1547,8 @@ class ParameterConverter { ), upscaleFactor: _asInt(v['upscaleFactor']) ?? 2, cuganNoise: v['cuganNoise'] as String? ?? 'conservative', + animejanaiModel: v['animejanaiModel'] as String? ?? 'compact', + animejanaiSharp: v['animejanaiSharp'] as bool? ?? false, // Absent (checkbox off) stays null so the script omits it entirely. bicubicB: (v['bicubicB'] as num?)?.toDouble(), bicubicC: (v['bicubicC'] as num?)?.toDouble(), diff --git a/app/lib/services/addon_service.dart b/app/lib/services/addon_service.dart new file mode 100644 index 00000000..f6c01fa5 --- /dev/null +++ b/app/lib/services/addon_service.dart @@ -0,0 +1,374 @@ +import 'dart:async'; +import 'dart:convert'; +import 'dart:io'; + +import 'package:archive/archive.dart'; +import 'package:crypto/crypto.dart'; +import 'package:flutter/foundation.dart'; +import 'package:flutter/services.dart'; +import 'package:path/path.dart' as path; + +import 'addon_manager.dart'; +import 'dependency_manager.dart'; +import 'whisper_addon_manager.dart'; + +/// One optional third-party download: something VapourBox can use but does not +/// ship, because of its size or its licence. Shown on Settings → Add-ons and +/// in the prompt a feature raises when it needs one. +class AddonInfo { + final String id; + + /// What the user knows it by. + final String name; + + /// The feature that uses it, grouping the Settings list ("Subtitles"). + final String feature; + final String description; + + /// SPDX-style licence name, and where to read it. + final String licence; + final String? licenceUrl; + + /// A restriction the user must be told before downloading, in one sentence + /// ("Free for non-commercial use only."). Null when the licence has none. + final String? usageNote; + + /// Who made it, as upstream states it. + final String credit; + + /// The heading of its entry in `licenses/NOTICES.txt` (and the start of its + /// About-dialog tile). `addon_service_test.dart` checks that entry states + /// the same licence and credit as this, so the Settings page cannot drift + /// from the notices. + final String noticeName; + + /// Where it comes from. + final String sourceUrl; + + /// Bytes downloaded to install it on this platform. + final int downloadBytes; + + /// Add-ons it cannot work without, installed first. Removing one of those + /// removes this too. + final List requires; + + /// Why it cannot be used on this platform, or null if it can. + final String? unavailableReason; + + const AddonInfo({ + required this.id, + required this.name, + required this.feature, + required this.description, + required this.licence, + this.licenceUrl, + this.usageNote, + required this.credit, + required this.noticeName, + required this.sourceUrl, + required this.downloadBytes, + this.requires = const [], + this.unavailableReason, + }); +} + +/// How one add-on is installed, recognised and removed under the add-ons +/// directory. Each must install atomically enough that [isInstalled] is never +/// true for a partial install. +abstract class AddonInstaller { + AddonInfo get info; + bool isInstalled(Directory addonsDir); + Future install(Directory addonsDir, AddonManager downloader); + Future remove(Directory addonsDir); +} + +/// Every optional download, in one place: what exists, what is installed, +/// installing (with the add-ons it requires) and removing (with the add-ons +/// that require it). The Settings → Add-ons page and the prompts features +/// raise ([AddonPrompt]) both go through this, so they cannot disagree. +class AddonService extends ChangeNotifier { + static final AddonService instance = AddonService._(); + AddonService._() : _downloader = AddonManager(); + + /// A service over [installers] in [addonsDir], downloading with + /// [downloader] — for tests. + @visibleForTesting + AddonService.forTest({ + required List installers, + required Directory addonsDir, + AddonManager? downloader, + }) : _downloader = downloader ?? AddonManager(), + _addonsDirOverride = addonsDir { + _setInstallers(installers); + } + + final AddonManager _downloader; + Directory? _addonsDirOverride; + final Map _installers = {}; + final Set _installed = {}; + Future? _loading; + StreamSubscription? _progressSub; + + /// The add-on being installed now, its progress (0–1, or null while it is + /// unknown) and status line. + String? installingId; + double? progress; + String status = ''; + + void _setInstallers(List installers) { + _installers + ..clear() + ..addEntries(installers.map((i) => MapEntry(i.info.id, i))); + } + + /// Read the catalogue and check what is installed. Safe to call repeatedly. + Future load() => _loading ??= _load(); + + Future _load() async { + if (_installers.isEmpty) { + final platform = DependencyManager.instance.platformId; + final whisper = WhisperAddonManager( + jsonDecode(await rootBundle.loadString('assets/whisper-addon.json')) + as Map, + platform, + ); + _setInstallers([ + ...whisper.installers(), + AnimeJanaiInstaller( + jsonDecode(await rootBundle.loadString('assets/animejanai-addon.json')) + as Map, + platform, + ), + ]); + } + await refresh(); + } + + /// Re-check what is installed (the user may have deleted files). + Future refresh() async { + final dir = await addonsDir(); + _installed + ..clear() + ..addAll(_installers.values.where((i) => i.isInstalled(dir)).map((i) => i.info.id)); + notifyListeners(); + } + + /// Every add-on, in catalogue order. + List get catalog => _installers.values.map((i) => i.info).toList(); + + AddonInfo? info(String id) => _installers[id]?.info; + + /// Whether [id] is installed, as of the last [refresh]. False before [load]. + bool isInstalled(String id) => _installed.contains(id); + + /// [ids] and everything they require that is not installed, requirements + /// first. + List missing(Iterable ids) { + final out = []; + void visit(String id) { + final installer = _installers[id]; + if (installer == null) throw ArgumentError('Unknown add-on: $id'); + for (final r in installer.info.requires) { + visit(r); + } + if (!isInstalled(id) && !out.contains(id)) out.add(id); + } + + ids.forEach(visit); + return out; + } + + /// Everything installed that would stop working if [id] were removed. + List dependants(String id) => _installers.values + .where((i) => i.info.requires.contains(id) && isInstalled(i.info.id)) + .expand((i) => [...dependants(i.info.id), i.info.id]) + .toSet() + .toList(); + + int downloadBytes(Iterable ids) => + ids.fold(0, (sum, id) => sum + (_installers[id]?.info.downloadBytes ?? 0)); + + /// Install [id] and whatever it requires. Returns false if cancelled; + /// throws on failure, leaving whatever had finished installed. + Future install(String id) async { + await load(); + if (installingId != null) throw StateError('Already installing $installingId'); + final dir = await addonsDir(); + try { + for (final next in missing([id])) { + final installer = _installers[next]!; + if (installer.info.unavailableReason != null) { + throw StateError(installer.info.unavailableReason!); + } + installingId = next; + progress = null; + status = 'Downloading ${installer.info.name}…'; + notifyListeners(); + _progressSub?.cancel(); + _progressSub = _downloader.progressStream.listen((p) { + progress = p.totalBytes > 0 ? p.progress : null; + notifyListeners(); + }); + await dir.create(recursive: true); + await installer.install(dir, _downloader); + if (_downloader.isCancelled) return false; + _installed.add(next); + } + return true; + } finally { + await _progressSub?.cancel(); + _progressSub = null; + installingId = null; + progress = null; + status = ''; + await refresh(); + } + } + + /// Stop the current download. A partial download is kept for resuming. + void cancel() => _downloader.cancelDownload(); + + /// Remove [id] and everything that requires it. + Future remove(String id) async { + final dir = await addonsDir(); + for (final d in [...dependants(id), id]) { + await _installers[d]!.remove(dir); + } + await refresh(); + } + + /// The add-ons directory — the same one the worker reads + /// (`DependencyLocator::addons_dir`): beside the executable on Windows, + /// Application Support on macOS, `$XDG_DATA_HOME` on Linux, and in a debug + /// build the nearest `addons/` above the executable. + Future addonsDir() async { + final override = _addonsDirOverride; + if (override != null) return override; + final appDir = path.dirname(Platform.resolvedExecutable); + if (kDebugMode) { + var current = Directory(appDir); + for (var i = 0; i < 12; i++) { + final candidate = Directory(path.join(current.path, 'addons')); + if (await candidate.exists()) return _addonsDirOverride = candidate; + final parent = current.parent; + if (parent.path == current.path) break; + current = parent; + } + } + final home = Platform.environment['HOME'] ?? '/tmp'; + if (Platform.isWindows) return Directory(path.join(appDir, 'addons')); + if (Platform.isMacOS) { + return Directory( + path.join(home, 'Library', 'Application Support', 'VapourBox', 'addons')); + } + final dataDir = + Platform.environment['XDG_DATA_HOME'] ?? path.join(home, '.local', 'share'); + return Directory(path.join(dataDir, 'VapourBox', 'addons')); + } +} + +/// AnimeJaNai's add-on id. +const animeJanaiAddonId = 'animejanai'; + +/// AnimeJaNai's models (see `worker/src/animejanai.rs`): upstream's +/// models-only zip, pinned by hash in `assets/animejanai-addon.json`, of which +/// only the ONNX models are kept, each checked against its own hash, into +/// `addons/animejanai/`. Built beside it and renamed into place, so a failed +/// or cancelled install never leaves a partial directory the worker would use. +class AnimeJanaiInstaller implements AddonInstaller { + final Map manifest; + final String platformId; + + AnimeJanaiInstaller(this.manifest, this.platformId); + + static const dirName = 'animejanai'; + + /// The platforms that ship VSNCNN, which runs these models + /// (`gpu_plugin::platform_ships_upscale`). + static const platforms = {'macos-arm64', 'windows-x64', 'linux-x64'}; + + Map get _models => + (manifest['models'] as Map).cast(); + + @override + AddonInfo get info { + final archive = manifest['archive'] as Map; + return AddonInfo( + id: animeJanaiAddonId, + name: 'AnimeJaNai upscaling models', + feature: 'AI upscaling', + description: 'Real-ESRGAN Compact models trained on HD anime, for Crop & ' + 'Resize\'s AnimeJaNai upscaler: the HD V3 models in three sizes, ' + 'each with a Sharp variant, and the SD V1 beta model.', + licence: manifest['licence'] as String, + licenceUrl: manifest['licenceUrl'] as String?, + usageNote: 'Free for non-commercial use only.', + credit: 'the-database (GitHub)', + noticeName: 'AnimeJaNai models', + sourceUrl: manifest['homepage'] as String, + downloadBytes: archive['size'] as int, + unavailableReason: platforms.contains(platformId) + ? null + : 'AnimeJaNai runs on the AI upscaler, which is not available on ' + 'this platform.', + ); + } + + @override + bool isInstalled(Directory addonsDir) => _models.keys + .every((m) => File(path.join(addonsDir.path, dirName, m)).existsSync()); + + @override + Future install(Directory addonsDir, AddonManager downloader) async { + final archive = manifest['archive'] as Map; + final target = Directory(path.join(addonsDir.path, dirName)); + final staging = Directory('${target.path}.new'); + try { + // Kept in the add-ons directory, not temp, so a cancelled download + // resumes (AddonManager keeps the .partial beside it). + final zip = File(path.join(addonsDir.path, '.animejanai-${manifest['version']}.zip')); + await downloader.downloadFile(archive['url'] as String, zip, + expectedSize: archive['size'] as int?, + expectedSha256: archive['sha256'] as String?); + if (downloader.isCancelled) return; + + if (await staging.exists()) await staging.delete(recursive: true); + await extractModels(await zip.readAsBytes(), _models, staging); + if (await target.exists()) await target.delete(recursive: true); + await staging.rename(target.path); + await zip.delete(); + } finally { + if (await staging.exists()) await staging.delete(recursive: true); + } + } + + /// Write each of [models] (name → SHA-256) from the zip [zipBytes] into + /// [into], refusing the lot if one is missing or does not match its hash. + /// Anything else in the zip (upstream's .pth training checkpoints) is left + /// out. Shared with the test harness, which installs the real download. + static Future extractModels( + List zipBytes, Map models, Directory into) async { + final entries = ZipDecoder().decodeBytes(zipBytes); + final found = >{}; + for (final MapEntry(key: name, value: hash) in models.entries) { + final entry = entries.files.where((f) => f.isFile && path.basename(f.name) == name); + if (entry.isEmpty) throw StateError('$name is missing from the AnimeJaNai download'); + final bytes = entry.first.content as List; + final actual = sha256.convert(bytes).toString(); + if (actual != hash) { + throw StateError('$name does not match its expected hash ($actual)'); + } + found[name] = bytes; + } + await into.create(recursive: true); + for (final MapEntry(key: name, value: bytes) in found.entries) { + await File(path.join(into.path, name)).writeAsBytes(bytes); + } + } + + @override + Future remove(Directory addonsDir) async { + final dir = Directory(path.join(addonsDir.path, dirName)); + if (await dir.exists()) await dir.delete(recursive: true); + } +} diff --git a/app/lib/services/whisper_addon_manager.dart b/app/lib/services/whisper_addon_manager.dart index e9e15aa1..8b9984d8 100644 --- a/app/lib/services/whisper_addon_manager.dart +++ b/app/lib/services/whisper_addon_manager.dart @@ -1,211 +1,115 @@ -import 'dart:async'; -import 'dart:convert'; import 'dart:io'; import 'package:archive/archive.dart'; -import 'package:flutter/foundation.dart'; -import 'package:flutter/services.dart'; import 'package:path/path.dart' as path; import 'addon_manager.dart'; +import 'addon_service.dart'; import 'temp_directory_service.dart'; -/// Manages Whisper add-on binary and model downloads. +/// The Whisper add-on, for subtitle generation: the whisper.cpp engine and its +/// language models, described by `assets/whisper-addon.json`, under +/// `addons/whisper/`. Each is an [AddonInstaller] for [AddonService]; every +/// model requires the engine. class WhisperAddonManager { - static final WhisperAddonManager instance = WhisperAddonManager._(); - WhisperAddonManager._(); + final Map metadata; + final String platformId; - final AddonManager _addonManager = AddonManager(); - Map? _metadata; + WhisperAddonManager(this.metadata, this.platformId); - /// Stream of download progress updates. - Stream get progressStream => _addonManager.progressStream; + static const dirName = 'whisper'; - /// Load metadata from bundled asset. - Future> _getMetadata() async { - if (_metadata != null) return _metadata!; - final jsonString = await rootBundle.loadString('assets/whisper-addon.json'); - _metadata = jsonDecode(jsonString) as Map; - return _metadata!; - } - - /// Get the current platform identifier. - String get _platformId { - if (Platform.isWindows) return 'windows-x64'; - if (Platform.isMacOS) { - final result = Process.runSync('uname', ['-m']); - final arch = result.stdout.toString().trim(); - return arch == 'arm64' ? 'macos-arm64' : 'macos-x64'; - } - if (Platform.isLinux) { - final result = Process.runSync('uname', ['-m']); - final arch = result.stdout.toString().trim(); - return arch == 'aarch64' ? 'linux-arm64' : 'linux-x64'; - } - throw UnsupportedError('Unsupported platform'); - } - - /// Get the whisper add-on directory. - Future getWhisperDir() async { - final base = await _getAddonsBaseDir(); - return Directory(path.join(base.path, 'whisper')); - } - - /// Get the add-ons base directory, mirroring deps directory location strategy. - Future _getAddonsBaseDir() async { - final executablePath = Platform.resolvedExecutable; - final appDir = path.dirname(executablePath); - - if (Platform.isWindows) { - // Production: addons folder next to executable - final prodAddons = Directory(path.join(appDir, 'addons')); - if (kDebugMode) { - // Development: search upward for project addons - var current = Directory(appDir); - for (var i = 0; i < 10; i++) { - final addonsDir = Directory(path.join(current.path, 'addons')); - if (await addonsDir.exists()) { - return addonsDir; - } - final parent = current.parent; - if (parent.path == current.path) break; - current = parent; - } - } - return prodAddons; - } else if (Platform.isMacOS) { - if (kDebugMode) { - // Development: search upward for project addons - var current = Directory(appDir); - for (var i = 0; i < 12; i++) { - final addonsDir = Directory(path.join(current.path, 'addons')); - if (await addonsDir.exists()) { - return addonsDir; - } - final parent = current.parent; - if (parent.path == current.path) break; - current = parent; - } - } - - // Production: Application Support - final home = Platform.environment['HOME'] ?? '/tmp'; - return Directory(path.join( - home, 'Library', 'Application Support', 'VapourBox', 'addons', - )); - } else if (Platform.isLinux) { - if (kDebugMode) { - // Development: search upward for project addons - var current = Directory(appDir); - for (var i = 0; i < 10; i++) { - final addonsDir = Directory(path.join(current.path, 'addons')); - if (await addonsDir.exists()) { - return addonsDir; - } - final parent = current.parent; - if (parent.path == current.path) break; - current = parent; - } - } - - // Production: XDG_DATA_HOME or ~/.local/share - final xdgDataHome = Platform.environment['XDG_DATA_HOME']; - final home = Platform.environment['HOME'] ?? '/tmp'; - final dataDir = xdgDataHome ?? path.join(home, '.local', 'share'); - return Directory(path.join(dataDir, 'VapourBox', 'addons')); - } - - throw UnsupportedError('Unsupported platform'); - } - - /// Whether the whisper binary is installed. - Future get isInstalled async { - final binaryPath = await whisperBinaryPath; - return File(binaryPath).existsSync(); - } - - /// Whether a specific model is installed. - Future isModelInstalled(String modelId) async { - final whisperDir = await getWhisperDir(); - return File(path.join(whisperDir.path, 'models', 'ggml-$modelId.bin')).existsSync(); - } + /// The add-on id of a language model (`small`, `medium`, `large-v3-turbo`). + static String modelAddonId(String modelId) => 'whisper-model-$modelId'; - /// Get path to the whisper binary. - /// - /// macOS: whisper/bin/whisper-cli (Homebrew bottle layout with lib/ sibling) - /// Windows: whisper/whisper-cli.exe (flat zip layout) - Future get whisperBinaryPath async { - final whisperDir = await getWhisperDir(); - if (Platform.isWindows) { - return path.join(whisperDir.path, 'whisper-cli.exe'); - } - return path.join(whisperDir.path, 'bin', 'whisper-cli'); - } + static const engineAddonId = 'whisper'; - /// Get path to a model file. - Future getModelPath(String modelId) async { - final whisperDir = await getWhisperDir(); - return path.join(whisperDir.path, 'models', 'ggml-$modelId.bin'); - } + List installers() => [ + _WhisperEngine(this), + for (final id in (metadata['models'] as Map).keys) + _WhisperModel(this, id), + ]; - /// Download and install the whisper binary. - Future downloadBinary() async { - final metadata = await _getMetadata(); - final binaryInfo = metadata['binary'] as Map; - final platforms = binaryInfo['platforms'] as Map; - final platformInfo = platforms[_platformId] as Map; - - final url = platformInfo['url'] as String; - final exeName = platformInfo['executable'] as String; - final expectedSize = platformInfo['size'] as int?; - final format = platformInfo['format'] as String? ?? 'zip'; + Map get _platformInfo => + ((metadata['binary'] as Map)['platforms'] + as Map)[platformId] as Map; +} - final whisperDir = await getWhisperDir(); +const _whisperCppCredit = + 'The ggml authors (project started by Georgi Gerganov)'; + +class _WhisperEngine implements AddonInstaller { + final WhisperAddonManager whisper; + _WhisperEngine(this.whisper); + + @override + AddonInfo get info => AddonInfo( + id: WhisperAddonManager.engineAddonId, + name: 'Whisper speech recognition (whisper.cpp)', + feature: 'Subtitles', + description: 'The engine that transcribes speech for the Subtitles ' + 'pass. Needs one of the language models below.', + licence: 'MIT', + credit: _whisperCppCredit, + noticeName: 'whisper.cpp', + sourceUrl: 'https://github.com/ggml-org/whisper.cpp', + downloadBytes: whisper._platformInfo['size'] as int? ?? 0, + ); + + /// macOS/Linux: whisper/bin/whisper-cli (with lib/ beside bin/ on macOS); + /// Windows: whisper/whisper-cli.exe — what `DependencyLocator::whisper_path` + /// looks for. + static String binaryPath(Directory addonsDir) { + final dir = path.join(addonsDir.path, WhisperAddonManager.dirName); + return Platform.isWindows + ? path.join(dir, 'whisper-cli.exe') + : path.join(dir, 'bin', 'whisper-cli'); + } + + @override + bool isInstalled(Directory addonsDir) => File(binaryPath(addonsDir)).existsSync(); + + @override + Future install(Directory addonsDir, AddonManager downloader) async { + final info = whisper._platformInfo; + final url = info['url'] as String; + final exeName = info['executable'] as String; + final format = info['format'] as String? ?? 'zip'; + final whisperDir = Directory(path.join(addonsDir.path, WhisperAddonManager.dirName)); await whisperDir.create(recursive: true); - final tempDir = - await TempDirectoryService.instance.createTemp('vapourbox_whisper_'); + final tempDir = await TempDirectoryService.instance.createTemp('vapourbox_whisper_'); final tempFile = File(path.join(tempDir.path, - (format == 'homebrew-bottle' || format == 'tar') - ? 'whisper.tar.gz' - : 'whisper.zip')); - + (format == 'homebrew-bottle' || format == 'tar') ? 'whisper.tar.gz' : 'whisper.zip')); try { // ghcr.io requires a bearer token (public, anonymous access) - Map? extraHeaders; - if (url.contains('ghcr.io')) { - extraHeaders = {'Authorization': 'Bearer QQ=='}; - } - - await _addonManager.downloadFile(url, tempFile, - expectedSize: expectedSize, extraHeaders: extraHeaders); - - if (_addonManager.isCancelled) return; + final extraHeaders = + url.contains('ghcr.io') ? {'Authorization': 'Bearer QQ=='} : null; + await downloader.downloadFile(url, tempFile, + expectedSize: info['size'] as int?, + expectedSha256: info['sha256'] as String?, + extraHeaders: extraHeaders); + if (downloader.isCancelled) return; if (format == 'homebrew-bottle') { - await _extractHomebrewBottle(tempFile, whisperDir, exeName); + // Bottle layout: whisper-cpp//{bin,lib}/ — keep bin/ and lib/ + // side by side so @rpath (@loader_path/../lib) resolves. + await _untar(tempFile, whisperDir, stripComponents: 2); } else if (format == 'tar') { - await _extractTar(tempFile, whisperDir, exeName); + // A static bin/whisper-cli, nothing to strip. + await _untar(tempFile, whisperDir); } else { await _extractZip(tempFile, whisperDir, exeName); } - // On macOS, remove quarantine and codesign all binaries/dylibs if (Platform.isMacOS) { + // Remove quarantine, and ad-hoc sign every binary and dylib. await Process.run('xattr', ['-cr', whisperDir.path]); - // Sign all Mach-O binaries and dylibs - final binDir = Directory(path.join(whisperDir.path, 'bin')); - if (await binDir.exists()) { - await for (final entity in binDir.list()) { - if (entity is File) { - await Process.run('codesign', ['--force', '--sign', '-', entity.path]); - } - } - } - final libDir = Directory(path.join(whisperDir.path, 'lib')); - if (await libDir.exists()) { - await for (final entity in libDir.list()) { - if (entity is File && entity.path.endsWith('.dylib')) { + for (final sub in ['bin', 'lib']) { + final dir = Directory(path.join(whisperDir.path, sub)); + if (!await dir.exists()) continue; + await for (final entity in dir.list()) { + if (entity is File && (sub == 'bin' || entity.path.endsWith('.dylib'))) { await Process.run('codesign', ['--force', '--sign', '-', entity.path]); } } @@ -218,121 +122,94 @@ class WhisperAddonManager { } } - /// Extract a Homebrew bottle (tar.gz) into the whisper directory. - /// - /// Bottle structure: whisper-cpp//bin/whisper-cli - /// whisper-cpp//lib/*.dylib - /// We extract bin/ and lib/ into whisperDir, preserving the subdirectory structure - /// so that @rpath (@loader_path/../lib) resolves correctly. - Future _extractHomebrewBottle( - File tarGz, Directory whisperDir, String exeName) async { - // Use system tar to extract (handles gzip + tar in one step) + Future _untar(File tarGz, Directory whisperDir, {int stripComponents = 0}) async { final result = await Process.run('tar', [ 'xzf', tarGz.path, '-C', whisperDir.path, - '--strip-components', '2', // Strip whisper-cpp// prefix + if (stripComponents > 0) ...['--strip-components', '$stripComponents'], ]); if (result.exitCode != 0) { - throw StateError('Failed to extract bottle: ${result.stderr}'); + throw StateError('Failed to extract whisper: ${result.stderr}'); } - - // Set executable permissions on extracted binaries final binDir = Directory(path.join(whisperDir.path, 'bin')); if (await binDir.exists()) { await for (final entity in binDir.list()) { - if (entity is File) { - await Process.run('chmod', ['+x', entity.path]); - } + if (entity is File) await Process.run('chmod', ['+x', entity.path]); } } } - /// Extract a plain gzipped tar (Linux binary release). - /// - /// The tarball is laid out as `bin/whisper-cli` (a fully static binary, no - /// `lib/` needed), matching the non-Windows whisperBinaryPath of - /// `whisper/bin/whisper-cli`. Unlike the Homebrew bottle, no path components - /// are stripped. - Future _extractTar( - File tarGz, Directory whisperDir, String exeName) async { - final result = await Process.run('tar', [ - 'xzf', tarGz.path, - '-C', whisperDir.path, - ]); - if (result.exitCode != 0) { - throw StateError('Failed to extract tarball: ${result.stderr}'); - } - - // Set executable permissions on extracted binaries - final binDir = Directory(path.join(whisperDir.path, 'bin')); - if (await binDir.exists()) { - await for (final entity in binDir.list()) { - if (entity is File) { - await Process.run('chmod', ['+x', entity.path]); - } - } - } - } - - /// Extract a zip archive (Windows binary release). - Future _extractZip( - File zipFile, Directory whisperDir, String exeName) async { - final bytes = await zipFile.readAsBytes(); - final archive = ZipDecoder().decodeBytes(bytes); - + /// The Windows release zip: keep the executable and its DLLs, flattened. + Future _extractZip(File zipFile, Directory whisperDir, String exeName) async { + final archive = ZipDecoder().decodeBytes(await zipFile.readAsBytes()); for (final file in archive) { - if (file.isFile) { - final fileName = path.basename(file.name); - if (fileName == exeName || fileName.endsWith('.dll') || fileName.endsWith('.dylib')) { - final outFile = File(path.join(whisperDir.path, fileName)); - await outFile.writeAsBytes(file.content as List); - } + if (!file.isFile) continue; + final fileName = path.basename(file.name); + if (fileName == exeName || fileName.endsWith('.dll') || fileName.endsWith('.dylib')) { + await File(path.join(whisperDir.path, fileName)) + .writeAsBytes(file.content as List); } } } - /// Download a whisper model. - Future downloadModel(String modelId) async { - final metadata = await _getMetadata(); - final models = metadata['models'] as Map; - final modelInfo = models[modelId] as Map; - - final url = modelInfo['url'] as String; - final filename = modelInfo['filename'] as String; - final expectedSize = modelInfo['size'] as int?; - - final whisperDir = await getWhisperDir(); - final modelsDir = Directory(path.join(whisperDir.path, 'models')); - await modelsDir.create(recursive: true); - - final destination = File(path.join(modelsDir.path, filename)); - await _addonManager.downloadFile(url, destination, expectedSize: expectedSize); - } - - /// Cancel the current download (preserves partial file for resume). - void cancelDownload() { - _addonManager.cancelDownload(); - } - - /// Whether the current download was cancelled. - bool get isCancelled => _addonManager.isCancelled; - - /// Get the total download size for binary + model. - Future getDownloadSize(String modelId) async { - final metadata = await _getMetadata(); - final binaryInfo = metadata['binary'] as Map; - final platforms = binaryInfo['platforms'] as Map; - final platformInfo = platforms[_platformId] as Map; - final binarySize = platformInfo['size'] as int? ?? 0; - - final models = metadata['models'] as Map; - final modelInfo = models[modelId] as Map; - final modelSize = modelInfo['size'] as int? ?? 0; - - return binarySize + modelSize; + /// The engine and everything beside it except the models, which are + /// removed as dependants first. + @override + Future remove(Directory addonsDir) async { + final dir = Directory(path.join(addonsDir.path, WhisperAddonManager.dirName)); + if (!await dir.exists()) return; + await for (final entity in dir.list()) { + if (path.basename(entity.path) != 'models') await entity.delete(recursive: true); + } } +} - void dispose() { - _addonManager.dispose(); +class _WhisperModel implements AddonInstaller { + final WhisperAddonManager whisper; + final String modelId; + _WhisperModel(this.whisper, this.modelId); + + Map get _model => + (whisper.metadata['models'] as Map)[modelId] as Map; + + static const _labels = { + 'small': 'Small (fastest)', + 'medium': 'Medium', + 'large-v3-turbo': 'Large v3 Turbo (most accurate)', + }; + + @override + AddonInfo get info => AddonInfo( + id: WhisperAddonManager.modelAddonId(modelId), + name: 'Whisper language model — ${_labels[modelId] ?? modelId}', + feature: 'Subtitles', + description: 'OpenAI\'s Whisper model in whisper.cpp\'s format, chosen ' + 'in the Subtitles pass. Larger models are slower and more accurate.', + licence: 'MIT', + credit: 'OpenAI', + noticeName: 'Whisper models', + sourceUrl: 'https://huggingface.co/ggerganov/whisper.cpp', + downloadBytes: _model['size'] as int? ?? 0, + requires: const [WhisperAddonManager.engineAddonId], + ); + + File _file(Directory addonsDir) => File(path.join(addonsDir.path, + WhisperAddonManager.dirName, 'models', _model['filename'] as String)); + + @override + bool isInstalled(Directory addonsDir) => _file(addonsDir).existsSync(); + + @override + Future install(Directory addonsDir, AddonManager downloader) async { + final file = _file(addonsDir); + await file.parent.create(recursive: true); + await downloader.downloadFile(_model['url'] as String, file, + expectedSize: _model['size'] as int?); + } + + @override + Future remove(Directory addonsDir) async { + final file = _file(addonsDir); + if (await file.exists()) await file.delete(); } } diff --git a/app/lib/views/about_dialog.dart b/app/lib/views/about_dialog.dart index 593cc904..28fef5f8 100644 --- a/app/lib/views/about_dialog.dart +++ b/app/lib/views/about_dialog.dart @@ -202,7 +202,7 @@ class _AboutDialogState extends State { ), _ComponentTile( name: 'RemoveGrain / Repair', - license: 'WTFPL', + license: 'WTFPL, with one MIT component', copyright: 'Laurent de Soras, Fredrik Mellbin; ' 'Clense by Victor Efimov (MIT)', url: 'https://github.com/vapoursynth/vs-removegrain', @@ -291,7 +291,8 @@ class _AboutDialogState extends State { license: 'GPL-3.0', copyright: 'AmusementClub (no copyright notice upstream); ' 'includes ncnn (Tencent), Protocol Buffers (Google, ' - 'BSD 3-Clause) and ONNX (Apache 2.0)', + 'BSD 3-Clause), ONNX (Apache 2.0) and, on macOS, the ' + 'LLVM OpenMP runtime (Apache 2.0 with LLVM exception)', url: 'https://github.com/AmusementClub/vs-mlrt', ), _ComponentTile( @@ -300,6 +301,12 @@ class _AboutDialogState extends State { copyright: 'Xintao Wang', url: 'https://github.com/xinntao/Real-ESRGAN', ), + _ComponentTile( + name: 'AnimeJaNai models (optional download)', + license: 'CC BY-NC-SA 4.0 (non-commercial use only)', + copyright: 'the-database', + url: 'https://github.com/the-database/mpv-upscale-2x_animejanai', + ), _ComponentTile( name: 'Real-CUGAN models', license: 'MIT', @@ -473,11 +480,17 @@ class _AboutDialogState extends State { url: 'https://ffmpeg.org', ), _ComponentTile( - name: 'whisper.cpp', + name: 'whisper.cpp (optional download)', license: 'MIT', - copyright: 'The ggml authors', + copyright: 'The ggml authors (project started by Georgi Gerganov)', url: 'https://github.com/ggml-org/whisper.cpp', ), + _ComponentTile( + name: 'Whisper models (optional download)', + license: 'MIT', + copyright: 'OpenAI', + url: 'https://github.com/openai/whisper', + ), _ComponentTile( name: 'libdvdread', license: 'GPL 2.0+', diff --git a/app/lib/views/addon_dialogs.dart b/app/lib/views/addon_dialogs.dart new file mode 100644 index 00000000..7d6ae214 --- /dev/null +++ b/app/lib/views/addon_dialogs.dart @@ -0,0 +1,224 @@ +import 'package:flutter/material.dart'; +import 'package:url_launcher/url_launcher.dart'; + +import '../services/addon_service.dart'; + +/// Human-readable size for a download. +String formatAddonSize(int bytes) { + if (bytes >= 1000000000) return '${(bytes / 1000000000).toStringAsFixed(1)} GB'; + if (bytes >= 1000000) return '${(bytes / 1000000).toStringAsFixed(0)} MB'; + return '${(bytes / 1000).toStringAsFixed(0)} KB'; +} + +/// What a feature calls when it needs add-ons: if any of [ids] (or what they +/// require) is missing, say what will be downloaded, from where and under +/// what licence, and download it on the user's say-so. Returns true when +/// everything is installed at the end. Settings → Add-ons installs through +/// the same [AddonService]. +class AddonPrompt { + static Future ensure( + BuildContext context, + List ids, { + required String reason, + AddonService? service, + }) async { + final addons = service ?? AddonService.instance; + await addons.load(); + final missing = addons.missing(ids); + if (missing.isEmpty) return true; + if (!context.mounted) return false; + final confirmed = await showDialog( + context: context, + barrierDismissible: false, + builder: (_) => AddonConfirmDialog(service: addons, ids: missing, reason: reason), + ); + if (confirmed != true || !context.mounted) return false; + final done = await showDialog( + context: context, + barrierDismissible: false, + builder: (_) => AddonProgressDialog(service: addons, ids: missing), + ); + return done == true; + } +} + +/// One add-on's licence and source, as both the prompt and the Settings page +/// show it. The usage restriction, when there is one, comes first and bold: +/// it is the one thing the user must not miss. +class AddonTerms extends StatelessWidget { + final AddonInfo info; + const AddonTerms({super.key, required this.info}); + + @override + Widget build(BuildContext context) { + final theme = Theme.of(context); + final small = theme.textTheme.bodySmall; + return Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + if (info.usageNote != null) + Text(info.usageNote!, + style: small?.copyWith(fontWeight: FontWeight.bold, color: theme.colorScheme.error)), + Wrap( + crossAxisAlignment: WrapCrossAlignment.center, + spacing: 4, + children: [ + Text('${info.credit} · ', style: small), + _Link(text: info.licence, url: info.licenceUrl, style: small), + Text('·', style: small), + _Link(text: 'Source', url: info.sourceUrl, style: small), + ], + ), + ], + ); + } +} + +class _Link extends StatelessWidget { + final String text; + final String? url; + final TextStyle? style; + const _Link({required this.text, required this.url, this.style}); + + @override + Widget build(BuildContext context) { + if (url == null) return Text(text, style: style); + return InkWell( + onTap: () => launchUrl(Uri.parse(url!)), + child: Text(text, + style: style?.copyWith( + color: Theme.of(context).colorScheme.primary, + decoration: TextDecoration.underline)), + ); + } +} + +class AddonConfirmDialog extends StatelessWidget { + final AddonService service; + final List ids; + final String reason; + + const AddonConfirmDialog({ + super.key, + required this.service, + required this.ids, + required this.reason, + }); + + @override + Widget build(BuildContext context) { + final infos = ids.map(service.info).whereType().toList(); + final unavailable = infos.where((i) => i.unavailableReason != null).toList(); + return AlertDialog( + title: const Text('Download add-on?'), + content: SizedBox( + width: 440, + child: Column( + mainAxisSize: MainAxisSize.min, + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + Text('$reason It is not included with VapourBox and is ' + 'downloaded from its own source:'), + for (final info in infos) ...[ + const SizedBox(height: 12), + Text('${info.name} (${formatAddonSize(info.downloadBytes)})', + style: Theme.of(context).textTheme.titleSmall), + AddonTerms(info: info), + ], + if (unavailable.isNotEmpty) ...[ + const SizedBox(height: 12), + Text(unavailable.first.unavailableReason!, + style: TextStyle(color: Theme.of(context).colorScheme.error)), + ], + const SizedBox(height: 12), + Text('You can remove it later in Settings → Add-ons.', + style: Theme.of(context).textTheme.bodySmall), + ], + ), + ), + actions: [ + TextButton( + onPressed: () => Navigator.of(context).pop(false), + child: const Text('Cancel'), + ), + FilledButton( + onPressed: unavailable.isEmpty ? () => Navigator.of(context).pop(true) : null, + child: Text('Download ${formatAddonSize(service.downloadBytes(ids))}'), + ), + ], + ); + } +} + +/// Installs [ids] in order, showing progress; pops true when all are in. +class AddonProgressDialog extends StatefulWidget { + final AddonService service; + final List ids; + const AddonProgressDialog({super.key, required this.service, required this.ids}); + + @override + State createState() => _AddonProgressDialogState(); +} + +class _AddonProgressDialogState extends State { + String? _error; + + @override + void initState() { + super.initState(); + _run(); + } + + Future _run() async { + setState(() => _error = null); + try { + for (final id in widget.ids) { + if (!await widget.service.install(id)) { + if (mounted) Navigator.of(context).pop(false); + return; + } + } + if (mounted) Navigator.of(context).pop(true); + } catch (e) { + if (mounted) setState(() => _error = e.toString()); + } + } + + @override + Widget build(BuildContext context) { + return AlertDialog( + title: const Text('Downloading add-on'), + content: SizedBox( + width: 400, + child: ListenableBuilder( + listenable: widget.service, + builder: (context, _) => Column( + mainAxisSize: MainAxisSize.min, + crossAxisAlignment: CrossAxisAlignment.start, + children: _error != null + ? [ + Text('Download failed', style: Theme.of(context).textTheme.titleSmall), + const SizedBox(height: 4), + SelectableText(_error!, style: Theme.of(context).textTheme.bodySmall), + ] + : [ + Text(widget.service.status.isEmpty ? 'Preparing…' : widget.service.status), + const SizedBox(height: 12), + LinearProgressIndicator(value: widget.service.progress), + ], + ), + ), + ), + actions: [ + TextButton( + onPressed: () { + if (_error == null) widget.service.cancel(); + Navigator.of(context).pop(false); + }, + child: const Text('Cancel'), + ), + if (_error != null) FilledButton(onPressed: _run, child: const Text('Retry')), + ], + ); + } +} diff --git a/app/lib/views/pass_list/pass_list_panel.dart b/app/lib/views/pass_list/pass_list_panel.dart index f67e61c7..d1aaf80b 100644 --- a/app/lib/views/pass_list/pass_list_panel.dart +++ b/app/lib/views/pass_list/pass_list_panel.dart @@ -8,7 +8,7 @@ import '../../services/whats_new_service.dart'; import '../../services/whisper_addon_manager.dart'; import '../../viewmodels/main_viewmodel.dart'; import '../pass_settings/pass_settings_inline.dart'; -import '../whisper_download_dialog.dart'; +import '../addon_dialogs.dart'; import 'pass_list_item.dart'; /// Panel showing the list of processing passes that can be enabled/disabled. @@ -273,21 +273,13 @@ class PassListPanel extends StatelessWidget { bool enabled, ) async { if (enabled) { - final isInstalled = await WhisperAddonManager.instance.isInstalled; final modelId = viewModel.processingPipeline.subtitles.model.value; - final modelInstalled = - await WhisperAddonManager.instance.isModelInstalled(modelId); - - if (!isInstalled || !modelInstalled) { - if (!context.mounted) return; - final confirmed = - await WhisperConfirmDialog.show(context, modelId: modelId); - if (!confirmed) return; - if (!context.mounted) return; - final downloaded = - await WhisperDownloadDialog.show(context, modelId: modelId); - if (!downloaded) return; - } + final ready = await AddonPrompt.ensure( + context, + [WhisperAddonManager.modelAddonId(modelId)], + reason: 'Subtitle generation needs the Whisper speech recognition add-on.', + ); + if (!ready) return; } viewModel.togglePass(PassType.subtitles, enabled); } diff --git a/app/lib/views/pass_settings/pass_settings_inline.dart b/app/lib/views/pass_settings/pass_settings_inline.dart index ff101612..d9d963ca 100644 --- a/app/lib/views/pass_settings/pass_settings_inline.dart +++ b/app/lib/views/pass_settings/pass_settings_inline.dart @@ -7,12 +7,13 @@ import '../../models/filter_schema.dart'; import '../../models/pass_advice.dart'; import '../../models/processing_pipeline.dart'; import '../../models/gpu_plugin_status.dart'; +import '../../services/addon_service.dart'; import '../../services/whisper_addon_manager.dart'; import '../../utils/pixel_format.dart'; import '../../viewmodels/main_viewmodel.dart'; import '../../widgets/warning_banner.dart'; import '../settings/dynamic_filter_panel.dart'; -import '../whisper_download_dialog.dart'; +import '../addon_dialogs.dart'; /// The settings for one processing pass, rendered inline underneath that pass's /// row in the pass list. Leads with the filter's description — what it does and @@ -101,6 +102,8 @@ class PassSettingsInline extends StatelessWidget { _buildOpenCLWarning(context, viewModel, filterId, params), _buildGpuPluginNotice(context, viewModel, filterId, params), _buildKnlmNotice(viewModel, filterId, params), + if (animeJanaiSelectedBy(filterId, params.values)) + const AnimeJanaiNotice(), if (filterId == 'descale') _buildDescaleDetect(context, viewModel), _buildBitDepthWarning(context, viewModel, schema, params), DynamicFilterPanelCompact( @@ -353,23 +356,36 @@ class PassSettingsInline extends StatelessWidget { (!oldParams.enabled || oldParams.values['model'] != newParams.values['model'])) { final newModelId = newParams.values['model'] as String? ?? 'medium'; - final addonInstalled = await WhisperAddonManager.instance.isInstalled; - final modelInstalled = - await WhisperAddonManager.instance.isModelInstalled(newModelId); - if (!addonInstalled || !modelInstalled) { - if (!context.mounted) return; - final confirmed = - await WhisperConfirmDialog.show(context, modelId: newModelId); - if (!confirmed) return; - if (!context.mounted) return; - final downloaded = - await WhisperDownloadDialog.show(context, modelId: newModelId); - if (!downloaded) return; - } + final ready = await AddonPrompt.ensure( + context, + [WhisperAddonManager.modelAddonId(newModelId)], + reason: 'Subtitle generation needs the Whisper speech recognition add-on.', + ); + if (!ready) return; + } + + // AnimeJaNai's models are an add-on: choosing it downloads them, after + // saying what they are and that they are for non-commercial use only. + if (_choosesAnimeJanai(filterId, oldParams, newParams)) { + if (!context.mounted) return; + final ready = await AddonPrompt.ensure( + context, + [animeJanaiAddonId], + reason: 'AnimeJaNai upscaling needs its models.', + ); + if (!ready) return; } viewModel.updateDynamicParams(filterId, newParams); } + bool _choosesAnimeJanai( + String filterId, + DynamicParameters oldParams, + DynamicParameters newParams, + ) => + animeJanaiSelectedBy(filterId, newParams.values) && + !animeJanaiSelectedBy(filterId, oldParams.values); + Widget _buildFallbackPanel(BuildContext context, PassType passType) { return Padding( padding: const EdgeInsets.symmetric(vertical: 16), @@ -526,3 +542,68 @@ class _FilterDescriptionState extends State<_FilterDescription> { ); } } + +/// Under Crop & Resize while AnimeJaNai is chosen: its terms, always — the +/// non-commercial restriction is the user's to keep in mind on every job — +/// and, if its models are not installed (deleted since, or chosen by a +/// preset), a button to download them. +class AnimeJanaiNotice extends StatefulWidget { + final AddonService? service; + const AnimeJanaiNotice({super.key, this.service}); + + @override + State createState() => _AnimeJanaiNoticeState(); +} + +class _AnimeJanaiNoticeState extends State { + AddonService get _service => widget.service ?? AddonService.instance; + + @override + void initState() { + super.initState(); + _service.load(); + } + + @override + Widget build(BuildContext context) { + return ListenableBuilder( + listenable: _service, + builder: (context, _) { + final info = _service.info(animeJanaiAddonId); + if (info == null) return const SizedBox.shrink(); + final installed = _service.isInstalled(animeJanaiAddonId); + return Padding( + padding: const EdgeInsets.only(bottom: 12), + child: Row( + children: [ + Expanded( + child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + Text( + installed + ? 'AnimeJaNai models installed.' + : 'AnimeJaNai\'s models are not installed.', + style: Theme.of(context).textTheme.bodySmall, + ), + AddonTerms(info: info), + ], + ), + ), + if (!installed) + OutlinedButton.icon( + onPressed: _service.installingId != null + ? null + : () => AddonPrompt.ensure(context, [animeJanaiAddonId], + reason: 'AnimeJaNai upscaling needs its models.', + service: _service), + icon: const Icon(Icons.download, size: 16), + label: Text('Download (${formatAddonSize(info.downloadBytes)})'), + ), + ], + ), + ); + }, + ); + } +} diff --git a/app/lib/views/settings/addons_settings_tab.dart b/app/lib/views/settings/addons_settings_tab.dart new file mode 100644 index 00000000..5b156d21 --- /dev/null +++ b/app/lib/views/settings/addons_settings_tab.dart @@ -0,0 +1,163 @@ +import 'package:flutter/material.dart'; + +import '../../services/addon_service.dart'; +import '../addon_dialogs.dart'; + +/// Settings → Add-ons: every optional download, grouped by the feature that +/// uses it, with its size, licence and source, installed or not, to install +/// or remove. Installing here and from a feature's prompt is the same +/// [AddonService] call, so both show the same state. +class AddonsSettingsTab extends StatefulWidget { + final AddonService? service; + const AddonsSettingsTab({super.key, this.service}); + + @override + State createState() => _AddonsSettingsTabState(); +} + +class _AddonsSettingsTabState extends State { + AddonService get _service => widget.service ?? AddonService.instance; + String? _error; + + @override + void initState() { + super.initState(); + _service.load().then((_) => _service.refresh()); + } + + Future _install(AddonInfo info) async { + setState(() => _error = null); + try { + await _service.install(info.id); + } catch (e) { + if (mounted) setState(() => _error = '${info.name}: $e'); + } + } + + Future _remove(AddonInfo info) async { + final also = _service.dependants(info.id).map((id) => _service.info(id)!.name).toList(); + final confirmed = await showDialog( + context: context, + builder: (context) => AlertDialog( + title: Text('Remove ${info.name}?'), + content: Text([ + 'Features that use it will ask to download it again.', + if (also.isNotEmpty) 'This also removes: ${also.join(', ')}.', + ].join('\n\n')), + actions: [ + TextButton( + onPressed: () => Navigator.of(context).pop(false), + child: const Text('Cancel')), + FilledButton( + onPressed: () => Navigator.of(context).pop(true), + child: const Text('Remove')), + ], + ), + ); + if (confirmed != true) return; + try { + await _service.remove(info.id); + } catch (e) { + if (mounted) setState(() => _error = '${info.name}: $e'); + } + } + + @override + Widget build(BuildContext context) { + final theme = Theme.of(context); + return ListenableBuilder( + listenable: _service, + builder: (context, _) { + final byFeature = >{}; + for (final info in _service.catalog) { + byFeature.putIfAbsent(info.feature, () => []).add(info); + } + return ListView( + padding: const EdgeInsets.all(16), + children: [ + Text( + 'Optional downloads from third parties, for features that need ' + 'them. Each is fetched from its own source when you install it ' + 'here or first use the feature.', + style: theme.textTheme.bodySmall, + ), + if (_error != null) ...[ + const SizedBox(height: 8), + Text(_error!, style: TextStyle(color: theme.colorScheme.error)), + ], + for (final MapEntry(key: feature, value: infos) in byFeature.entries) ...[ + const SizedBox(height: 16), + Text(feature, style: theme.textTheme.titleSmall), + const SizedBox(height: 4), + for (final info in infos) _row(context, info), + ], + ], + ); + }, + ); + } + + Widget _row(BuildContext context, AddonInfo info) { + final theme = Theme.of(context); + final installed = _service.isInstalled(info.id); + final installing = _service.installingId == info.id; + final busy = _service.installingId != null; + final Widget action; + if (installing) { + action = TextButton(onPressed: _service.cancel, child: const Text('Cancel')); + } else if (installed) { + action = OutlinedButton( + key: ValueKey('remove-${info.id}'), + onPressed: busy ? null : () => _remove(info), + child: const Text('Remove'), + ); + } else { + action = FilledButton.tonal( + key: ValueKey('install-${info.id}'), + onPressed: busy || info.unavailableReason != null ? null : () => _install(info), + child: const Text('Install'), + ); + } + return Card( + margin: const EdgeInsets.only(bottom: 8), + child: Padding( + padding: const EdgeInsets.all(12), + child: Row( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + Expanded( + child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + Row( + children: [ + Flexible(child: Text(info.name, style: theme.textTheme.titleSmall)), + const SizedBox(width: 8), + Text( + installed ? 'Installed' : formatAddonSize(info.downloadBytes), + style: theme.textTheme.bodySmall?.copyWith( + color: installed ? theme.colorScheme.primary : null), + ), + ], + ), + const SizedBox(height: 2), + Text(info.description, style: theme.textTheme.bodySmall), + const SizedBox(height: 4), + AddonTerms(info: info), + if (info.unavailableReason != null) + Text(info.unavailableReason!, style: theme.textTheme.bodySmall), + if (installing) ...[ + const SizedBox(height: 8), + LinearProgressIndicator(value: _service.progress), + ], + ], + ), + ), + const SizedBox(width: 12), + action, + ], + ), + ), + ); + } +} diff --git a/app/lib/views/settings/settings_dialog.dart b/app/lib/views/settings/settings_dialog.dart index 4c1fdd23..753b13a1 100644 --- a/app/lib/views/settings/settings_dialog.dart +++ b/app/lib/views/settings/settings_dialog.dart @@ -22,6 +22,7 @@ import '../../viewmodels/main_viewmodel.dart'; import '../../widgets/new_badge.dart'; import '../../widgets/release_notes_text.dart'; import '../../widgets/warning_banner.dart'; +import 'addons_settings_tab.dart'; /// The output colour format explanation, shown in a scrollable dialog. /// @@ -315,7 +316,7 @@ class _SettingsDialogState extends State @override void initState() { super.initState(); - _tabController = TabController(length: 4, vsync: this); + _tabController = TabController(length: 5, vsync: this); } @override @@ -367,6 +368,7 @@ class _SettingsDialogState extends State Tab(text: 'General'), Tab(text: 'Output'), Tab(text: 'Input'), + Tab(text: 'Add-ons'), Tab(text: 'Changes'), ], ), @@ -379,6 +381,7 @@ class _SettingsDialogState extends State _GeneralSettingsTab(), _OutputSettingsTab(), _InputSettingsTab(), + AddonsSettingsTab(), _ChangesSettingsTab(), ], ), diff --git a/app/lib/views/settings/widgets/parameter_widgets.dart b/app/lib/views/settings/widgets/parameter_widgets.dart index 7146d559..8e3e710a 100644 --- a/app/lib/views/settings/widgets/parameter_widgets.dart +++ b/app/lib/views/settings/widgets/parameter_widgets.dart @@ -333,7 +333,7 @@ class _DropdownParameterWidget extends StatelessWidget { items: options.map((option) { return DropdownMenuItem( value: option, - child: Text(_formatOptionName(option)), + child: Text(param.ui?.optionLabels?[option] ?? _formatOptionName(option)), ); }).toList(), onChanged: (newValue) { diff --git a/app/lib/views/whisper_download_dialog.dart b/app/lib/views/whisper_download_dialog.dart deleted file mode 100644 index 6de72344..00000000 --- a/app/lib/views/whisper_download_dialog.dart +++ /dev/null @@ -1,239 +0,0 @@ -import 'dart:async'; - -import 'package:flutter/material.dart'; - -import '../services/addon_manager.dart'; -import '../services/whisper_addon_manager.dart'; - -/// Confirmation dialog shown when user first enables subtitles. -class WhisperConfirmDialog extends StatelessWidget { - final String modelId; - final int totalSize; - - const WhisperConfirmDialog({ - super.key, - required this.modelId, - required this.totalSize, - }); - - /// Show the confirmation dialog. Returns true if user wants to proceed. - static Future show(BuildContext context, {String modelId = 'medium'}) async { - final totalSize = await WhisperAddonManager.instance.getDownloadSize(modelId); - if (!context.mounted) return false; - - final result = await showDialog( - context: context, - barrierDismissible: false, - builder: (context) => WhisperConfirmDialog( - modelId: modelId, - totalSize: totalSize, - ), - ); - return result ?? false; - } - - String _formatSize(int bytes) { - if (bytes >= 1000000000) { - return '${(bytes / 1000000000).toStringAsFixed(1)} GB'; - } else if (bytes >= 1000000) { - return '${(bytes / 1000000).toStringAsFixed(0)} MB'; - } - return '${(bytes / 1000).toStringAsFixed(0)} KB'; - } - - @override - Widget build(BuildContext context) { - return AlertDialog( - title: const Text('Download Whisper AI'), - content: Column( - mainAxisSize: MainAxisSize.min, - crossAxisAlignment: CrossAxisAlignment.start, - children: [ - const Text( - 'Subtitle generation requires the Whisper AI add-on. ' - 'This will download the whisper binary and language model.', - ), - const SizedBox(height: 16), - Text( - 'Total download: ${_formatSize(totalSize)}', - style: Theme.of(context).textTheme.bodyMedium?.copyWith( - fontWeight: FontWeight.w600, - ), - ), - ], - ), - actions: [ - TextButton( - onPressed: () => Navigator.of(context).pop(false), - child: const Text('Cancel'), - ), - FilledButton( - onPressed: () => Navigator.of(context).pop(true), - child: const Text('Download & Enable'), - ), - ], - ); - } -} - -/// Progress dialog for Whisper download. -class WhisperDownloadDialog extends StatefulWidget { - final String modelId; - - const WhisperDownloadDialog({super.key, required this.modelId}); - - /// Show the download dialog. Returns true if download completed successfully. - static Future show(BuildContext context, {String modelId = 'medium'}) async { - final result = await showDialog( - context: context, - barrierDismissible: false, - builder: (context) => WhisperDownloadDialog(modelId: modelId), - ); - return result ?? false; - } - - @override - State createState() => _WhisperDownloadDialogState(); -} - -class _WhisperDownloadDialogState extends State { - final _manager = WhisperAddonManager.instance; - StreamSubscription? _progressSub; - - String _status = 'Preparing...'; - double _progress = 0.0; - String? _error; - - @override - void initState() { - super.initState(); - _startDownload(); - } - - @override - void dispose() { - _progressSub?.cancel(); - super.dispose(); - } - - Future _startDownload() async { - setState(() { - _error = null; - _status = 'Downloading Whisper binary...'; - }); - - _progressSub?.cancel(); - _progressSub = _manager.progressStream.listen((progress) { - if (mounted) { - setState(() { - _progress = progress.progress; - _status = progress.status; - }); - } - }); - - try { - // Phase 1: Download binary - final isInstalled = await _manager.isInstalled; - if (!isInstalled) { - await _manager.downloadBinary(); - if (_manager.isCancelled) { - if (mounted) Navigator.of(context).pop(false); - return; - } - } - - // Phase 2: Download model - if (mounted) { - setState(() { - _status = 'Downloading language model...'; - _progress = 0.0; - }); - } - - final modelInstalled = await _manager.isModelInstalled(widget.modelId); - if (!modelInstalled) { - await _manager.downloadModel(widget.modelId); - if (_manager.isCancelled) { - if (mounted) Navigator.of(context).pop(false); - return; - } - } - - if (mounted) { - Navigator.of(context).pop(true); - } - } catch (e) { - if (mounted) { - setState(() { - _error = e.toString(); - }); - } - } - } - - @override - Widget build(BuildContext context) { - return AlertDialog( - title: const Text('Downloading Whisper AI'), - content: SizedBox( - width: 400, - child: Column( - mainAxisSize: MainAxisSize.min, - crossAxisAlignment: CrossAxisAlignment.start, - children: [ - if (_error != null) ...[ - Icon( - Icons.error_outline, - size: 48, - color: Theme.of(context).colorScheme.error, - ), - const SizedBox(height: 8), - Text( - 'Download failed', - style: Theme.of(context).textTheme.titleSmall, - ), - const SizedBox(height: 4), - Text( - _error!, - style: Theme.of(context).textTheme.bodySmall, - maxLines: 3, - overflow: TextOverflow.ellipsis, - ), - ] else ...[ - Text(_status), - const SizedBox(height: 12), - LinearProgressIndicator(value: _progress > 0 ? _progress : null), - const SizedBox(height: 8), - if (_progress > 0) - Text( - '${(_progress * 100).toStringAsFixed(1)}%', - style: Theme.of(context).textTheme.bodySmall, - ), - ], - ], - ), - ), - actions: [ - if (_error != null) ...[ - TextButton( - onPressed: () => Navigator.of(context).pop(false), - child: const Text('Cancel'), - ), - FilledButton( - onPressed: _startDownload, - child: const Text('Retry'), - ), - ] else ...[ - TextButton( - onPressed: () { - _manager.cancelDownload(); - Navigator.of(context).pop(false); - }, - child: const Text('Cancel'), - ), - ], - ], - ); - } -} diff --git a/app/pubspec.yaml b/app/pubspec.yaml index d0591d71..8290ef12 100644 --- a/app/pubspec.yaml +++ b/app/pubspec.yaml @@ -57,5 +57,6 @@ flutter: assets: - assets/deps-version.json - assets/whisper-addon.json + - assets/animejanai-addon.json - assets/filters/ - assets/filters/core/ diff --git a/app/test/addon_service_test.dart b/app/test/addon_service_test.dart new file mode 100644 index 00000000..a62b2517 --- /dev/null +++ b/app/test/addon_service_test.dart @@ -0,0 +1,297 @@ +// AddonService: what the Settings → Add-ons page and every feature's download +// prompt go through. Checked here without the network: requirements install +// first and are removed with what needs them; AnimeJaNai's installer keeps +// only the hashed models and leaves nothing half-installed when a hash is +// wrong; and both shipped manifests describe what the code expects. +import 'dart:convert'; +import 'dart:io'; + +import 'package:archive/archive.dart'; +import 'package:crypto/crypto.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:path/path.dart' as p; +import 'package:vapourbox/services/addon_manager.dart'; +import 'package:vapourbox/services/addon_service.dart'; +import 'package:vapourbox/services/whisper_addon_manager.dart'; + +/// An add-on that is a single marker file. +class _Fake implements AddonInstaller { + @override + final AddonInfo info; + final List log; + _Fake(String id, this.log, {List requires = const [], String? unavailable}) + : info = AddonInfo( + id: id, + name: 'Fake $id', + feature: 'Test', + description: '', + licence: 'MIT', + credit: 'tests', + noticeName: 'Test', + sourceUrl: 'https://example.invalid', + downloadBytes: 10, + requires: requires, + unavailableReason: unavailable, + ); + + File _f(Directory d) => File(p.join(d.path, info.id)); + @override + bool isInstalled(Directory d) => _f(d).existsSync(); + @override + Future install(Directory d, AddonManager _) async { + log.add('install ${info.id}'); + await _f(d).writeAsString('x'); + } + + @override + Future remove(Directory d) async { + log.add('remove ${info.id}'); + await _f(d).delete(); + } +} + +/// Serves one fixed body for every download, after checking its hash the way +/// AddonManager does. +class _ServeBytes extends AddonManager { + final List body; + _ServeBytes(this.body); + @override + Future downloadFile(String url, File destination, + {int? expectedSize, String? expectedSha256, Map? extraHeaders}) async { + if (expectedSha256 != null && sha256.convert(body).toString() != expectedSha256) { + throw StateError('SHA256 mismatch'); + } + await destination.writeAsBytes(body); + } +} + +String _repoFile(String rel) => File(p.join( + Directory.current.path.endsWith('app') ? Directory.current.parent.path : Directory.current.path, + rel)) + .readAsStringSync(); + +void main() { + late Directory tmp; + setUp(() => tmp = Directory.systemTemp.createTempSync('vb_addons_')); + tearDown(() => tmp.deleteSync(recursive: true)); + + group('requirements', () { + late List log; + late AddonService service; + setUp(() async { + log = []; + service = AddonService.forTest(addonsDir: tmp, installers: [ + _Fake('engine', log), + _Fake('model-a', log, requires: ['engine']), + _Fake('model-b', log, requires: ['engine']), + _Fake('elsewhere', log, unavailable: 'Not on this platform.'), + ]); + await service.load(); + }); + + test('installing something installs what it requires first', () async { + expect(service.missing(['model-a']), ['engine', 'model-a']); + expect(service.downloadBytes(service.missing(['model-a'])), 20); + expect(await service.install('model-a'), isTrue); + expect(log, ['install engine', 'install model-a']); + expect(service.isInstalled('engine'), isTrue); + expect(service.missing(['model-b']), ['model-b'], reason: 'engine is already in'); + }); + + test('removing something removes what requires it', () async { + await service.install('model-a'); + await service.install('model-b'); + log.clear(); + expect(service.dependants('engine').toSet(), {'model-a', 'model-b'}); + await service.remove('engine'); + expect(log.last, 'remove engine'); + expect(log.toSet(), {'remove model-a', 'remove model-b', 'remove engine'}); + expect(service.catalog.where((i) => service.isInstalled(i.id)), isEmpty); + }); + + test('an add-on unavailable here is refused, not attempted', () async { + await expectLater(service.install('elsewhere'), throwsStateError); + expect(log, isEmpty); + expect(service.installingId, isNull, reason: 'a failed install must not stay busy'); + }); + + test('refresh notices files removed behind its back', () async { + await service.install('engine'); + File(p.join(tmp.path, 'engine')).deleteSync(); + await service.refresh(); + expect(service.isInstalled('engine'), isFalse); + }); + }); + + group('AnimeJaNai installer', () { + final models = {'a.onnx': utf8.encode('model a'), 'b.onnx': utf8.encode('model b')}; + List zipOf(Map> files) { + final archive = Archive(); + for (final MapEntry(key: name, value: bytes) in files.entries) { + archive.addFile(ArchiveFile('2x_ModelsOnly/$name', bytes.length, bytes)); + } + return ZipEncoder().encode(archive)!; + } + + Map manifest(List zip, Map> hashed) => { + 'version': 'test', + 'licence': 'CC BY-NC-SA 4.0', + 'homepage': 'https://example.invalid', + 'archive': { + 'url': 'https://example.invalid/m.zip', + 'size': zip.length, + 'sha256': sha256.convert(zip).toString(), + }, + 'models': { + for (final MapEntry(key: n, value: b) in hashed.entries) n: sha256.convert(b).toString() + }, + }; + + test('keeps only the listed models, each hash-checked, then reports installed', () async { + final zip = zipOf({...models, 'a.pth': utf8.encode('checkpoint')}); + final installer = AnimeJanaiInstaller(manifest(zip, models), 'macos-arm64'); + final service = AddonService.forTest( + addonsDir: tmp, installers: [installer], downloader: _ServeBytes(zip)); + await service.load(); + expect(service.isInstalled(animeJanaiAddonId), isFalse); + expect(await service.install(animeJanaiAddonId), isTrue); + final dir = Directory(p.join(tmp.path, AnimeJanaiInstaller.dirName)); + expect(dir.listSync().map((f) => p.basename(f.path)).toSet(), {'a.onnx', 'b.onnx'}); + expect(service.isInstalled(animeJanaiAddonId), isTrue); + expect(tmp.listSync().map((f) => p.basename(f.path)), [AnimeJanaiInstaller.dirName], + reason: 'the download and staging directory are cleaned up'); + await service.remove(animeJanaiAddonId); + expect(dir.existsSync(), isFalse); + }); + + test('a model with the wrong hash fails the install and leaves nothing behind', () async { + final zip = zipOf({'a.onnx': models['a.onnx']!, 'b.onnx': utf8.encode('tampered')}); + final installer = AnimeJanaiInstaller(manifest(zip, models), 'macos-arm64'); + final service = AddonService.forTest( + addonsDir: tmp, installers: [installer], downloader: _ServeBytes(zip)); + await service.load(); + await expectLater(service.install(animeJanaiAddonId), throwsStateError); + expect(Directory(p.join(tmp.path, AnimeJanaiInstaller.dirName)).existsSync(), isFalse); + expect(Directory(p.join(tmp.path, '${AnimeJanaiInstaller.dirName}.new')).existsSync(), isFalse); + expect(service.isInstalled(animeJanaiAddonId), isFalse); + }); + + test('a missing model fails the install', () async { + final zip = zipOf({'a.onnx': models['a.onnx']!}); + final installer = AnimeJanaiInstaller(manifest(zip, models), 'macos-arm64'); + final service = AddonService.forTest( + addonsDir: tmp, installers: [installer], downloader: _ServeBytes(zip)); + await service.load(); + await expectLater(service.install(animeJanaiAddonId), + throwsA(predicate((e) => '$e'.contains('b.onnx is missing')))); + }); + + test('is unavailable where the AI upscaler does not ship', () { + final zip = zipOf(models); + for (final platform in ['macos-x64', 'linux-arm64']) { + expect(AnimeJanaiInstaller(manifest(zip, models), platform).info.unavailableReason, + isNotNull, reason: platform); + } + for (final platform in AnimeJanaiInstaller.platforms) { + expect(AnimeJanaiInstaller(manifest(zip, models), platform).info.unavailableReason, + isNull, reason: platform); + } + }); + }); + + group('shipped manifests', () { + test('AnimeJaNai: a pinned upstream release asset, hashed, non-commercial, as the worker expects', () { + final m = jsonDecode(_repoFile('app/assets/animejanai-addon.json')) as Map; + final archive = m['archive'] as Map; + expect(archive['url'], startsWith( + 'https://github.com/the-database/mpv-upscale-2x_animejanai/releases/download/')); + expect(archive['sha256'], matches(RegExp(r'^[0-9a-f]{64}$'))); + expect(archive['size'], greaterThan(1000000)); + expect(m['licence'], 'CC BY-NC-SA 4.0'); + final info = AnimeJanaiInstaller(m, 'macos-arm64').info; + expect(info.usageNote, contains('non-commercial')); + final models = (m['models'] as Map); + for (final hash in models.values) { + expect(hash, matches(RegExp(r'^[0-9a-f]{64}$'))); + } + // The worker's list (worker/src/animejanai.rs MODELS), so the app never + // reports installed a set the worker would call incomplete. + final rust = _repoFile('worker/src/animejanai.rs'); + final listed = RegExp(r'^pub const MODELS: &\[&str\] = &\[(.*?)\];', multiLine: true, dotAll: true) + .firstMatch(rust)! + .group(1)!; + final workerModels = + RegExp(r'"([^"]+\.onnx)"').allMatches(listed).map((x) => x.group(1)).toSet(); + expect(models.keys.toSet(), workerModels); + }); + + test('Whisper: every platform has an engine, every model needs it', () { + final m = jsonDecode(_repoFile('app/assets/whisper-addon.json')) as Map; + for (final platform in ['macos-arm64', 'macos-x64', 'windows-x64', 'linux-x64', 'linux-arm64']) { + final installers = WhisperAddonManager(m, platform).installers(); + final engine = installers.first.info; + expect(engine.id, WhisperAddonManager.engineAddonId); + expect(engine.downloadBytes, greaterThan(0), reason: platform); + final models = installers.skip(1).map((i) => i.info).toList(); + expect(models.map((i) => i.id), containsAll([ + WhisperAddonManager.modelAddonId('small'), + WhisperAddonManager.modelAddonId('medium'), + WhisperAddonManager.modelAddonId('large-v3-turbo'), + ])); + for (final model in models) { + expect(model.requires, [WhisperAddonManager.engineAddonId]); + expect(model.downloadBytes, greaterThan(100000000)); + } + } + }); + }); + + // The Settings → Add-ons page states each add-on's licence and credit from + // its AddonInfo; the same facts are in NOTICES, the About dialog and the + // README. All four must agree, and a non-commercial licence must carry its + // usage note. + group('credits agree everywhere', () { + final notices = _repoFile('licenses/NOTICES.txt'); + final about = _repoFile('app/lib/views/about_dialog.dart'); + final readme = _repoFile('README.md'); + final infos = [ + for (final platform in ['macos-arm64', 'windows-x64', 'linux-x64']) + ...WhisperAddonManager( + jsonDecode(_repoFile('app/assets/whisper-addon.json')) as Map, + platform) + .installers() + .map((i) => i.info), + AnimeJanaiInstaller( + jsonDecode(_repoFile('app/assets/animejanai-addon.json')) as Map, + 'macos-arm64') + .info, + ]; + final byNotice = {for (final i in infos) '${i.noticeName}|${i.licence}|${i.credit}': i}.values; + + for (final info in byNotice) { + test(info.noticeName, () { + final heading = RegExp('^${RegExp.escape(info.noticeName)} — (.*)\$', multiLine: true) + .firstMatch(notices); + expect(heading, isNotNull, reason: 'NOTICES.txt has no "${info.noticeName} — " entry'); + expect(heading!.group(1), startsWith(info.licence), + reason: 'NOTICES and Settings → Add-ons state different licences'); + final body = notices.substring(heading.end).split(RegExp(r'\n\s*\n')).first; + expect(body, contains(info.credit), reason: 'NOTICES credits someone else'); + expect(body, contains('add-on'), reason: 'NOTICES must say it is not bundled'); + + final tile = RegExp("name: '${RegExp.escape(info.noticeName)}[^']*',\\s*license: '([^']+)'," + "\\s*copyright: '([^']+)'") + .firstMatch(about); + expect(tile, isNotNull, reason: 'the About dialog has no "${info.noticeName}" tile'); + expect(tile!.group(1), startsWith(info.licence)); + + final who = info.credit.split(' (').first.toLowerCase(); + final acks = readme.substring(readme.indexOf('## Acknowledgments')); + expect(acks.toLowerCase(), contains(who), reason: 'README Acknowledgments omit $who'); + + expect(info.usageNote != null, info.licence.contains('NC'), + reason: 'a non-commercial licence must be shown as a usage note, and only that'); + }); + } + }); +} diff --git a/app/test/addons_ui_widget_test.dart b/app/test/addons_ui_widget_test.dart new file mode 100644 index 00000000..017f547a --- /dev/null +++ b/app/test/addons_ui_widget_test.dart @@ -0,0 +1,120 @@ +// Settings → Add-ons and the AnimeJaNai notice, over a test AddonService: each +// add-on shows its size, licence and any usage restriction; Install and Remove +// do what they say (Remove naming what goes with it); and the Crop & Resize +// notice keeps the non-commercial terms on screen, offering the download only +// while the models are missing. +import 'dart:io'; + +import 'package:flutter/material.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:path/path.dart' as p; +import 'package:vapourbox/services/addon_manager.dart'; +import 'package:vapourbox/services/addon_service.dart'; +import 'package:vapourbox/views/pass_settings/pass_settings_inline.dart'; +import 'package:vapourbox/views/settings/addons_settings_tab.dart'; + +class _Marker implements AddonInstaller { + @override + final AddonInfo info; + _Marker(this.info); + File _f(Directory d) => File(p.join(d.path, info.id)); + @override + bool isInstalled(Directory d) => _f(d).existsSync(); + @override + Future install(Directory d, AddonManager _) => _f(d).writeAsString('x'); + @override + Future remove(Directory d) => _f(d).delete(); +} + +AddonInfo _info(String id, String feature, + {String? usageNote, List requires = const [], int bytes = 25000000}) => + AddonInfo( + id: id, + name: 'Name of $id', + feature: feature, + description: 'Description of $id', + licence: id == animeJanaiAddonId ? 'CC BY-NC-SA 4.0' : 'MIT', + usageNote: usageNote, + credit: 'Credit', + noticeName: 'Test', + sourceUrl: 'https://example.invalid/$id', + downloadBytes: bytes, + requires: requires, + ); + +void main() { + late Directory tmp; + late AddonService service; + + setUp(() async { + tmp = Directory.systemTemp.createTempSync('vb_addons_ui_'); + service = AddonService.forTest(addonsDir: tmp, installers: [ + _Marker(_info('whisper', 'Subtitles', bytes: 3400000)), + _Marker(_info('whisper-model-small', 'Subtitles', requires: ['whisper'], bytes: 466000000)), + _Marker(_info(animeJanaiAddonId, 'AI upscaling', + usageNote: 'Free for non-commercial use only.')), + ]); + await service.load(); + }); + tearDown(() => tmp.deleteSync(recursive: true)); + + Future pump(WidgetTester tester, Widget child) async { + await tester.pumpWidget(MaterialApp(home: Scaffold(body: child))); + await tester.pumpAndSettle(); + } + + testWidgets('lists every add-on by feature with size, licence and restriction', (tester) async { + await pump(tester, AddonsSettingsTab(service: service)); + expect(find.text('Subtitles'), findsOneWidget); + expect(find.text('AI upscaling'), findsOneWidget); + expect(find.text('Name of whisper-model-small'), findsOneWidget); + expect(find.text('466 MB'), findsOneWidget); + expect(find.text('CC BY-NC-SA 4.0'), findsOneWidget); + expect(find.text('Free for non-commercial use only.'), findsOneWidget); + expect(find.text('Install'), findsNWidgets(3)); + }); + + testWidgets('Install installs, then offers Remove; Remove names what goes with it', + (tester) async { + await pump(tester, AddonsSettingsTab(service: service)); + await tester.runAsync(() async { + await tester.tap(find.byKey(const ValueKey('install-whisper-model-small'))); + await Future.delayed(const Duration(milliseconds: 200)); + }); + await tester.pumpAndSettle(); + expect(service.isInstalled('whisper'), isTrue, reason: 'its requirement came first'); + expect(service.isInstalled('whisper-model-small'), isTrue); + expect(find.byKey(const ValueKey('remove-whisper')), findsOneWidget); + expect(find.text('Installed'), findsNWidgets(2)); + + await tester.tap(find.byKey(const ValueKey('remove-whisper'))); + await tester.pumpAndSettle(); + expect(find.textContaining('This also removes: Name of whisper-model-small'), findsOneWidget); + await tester.tap(find.widgetWithText(FilledButton, 'Remove')); + await tester.pumpAndSettle(); + // The removal itself is real file I/O, one step per file, after the + // dialog closes: let each step's I/O complete and its continuation run. + for (var i = 0; i < 10 && service.isInstalled('whisper'); i++) { + await tester.runAsync(() => Future.delayed(const Duration(milliseconds: 50))); + await tester.pump(); + } + await tester.pumpAndSettle(); + expect(service.isInstalled('whisper'), isFalse); + expect(service.isInstalled('whisper-model-small'), isFalse); + }); + + testWidgets('the AnimeJaNai notice shows its terms, and Download only while missing', + (tester) async { + await pump(tester, AnimeJanaiNotice(service: service)); + expect(find.text('Free for non-commercial use only.'), findsOneWidget); + expect(find.textContaining('not installed'), findsOneWidget); + expect(find.textContaining('Download (25 MB)'), findsOneWidget); + + await tester.runAsync(() => service.install(animeJanaiAddonId)); + await tester.pumpAndSettle(); + expect(find.text('Free for non-commercial use only.'), findsOneWidget, + reason: 'the restriction applies to every job, not just the download'); + expect(find.textContaining('Download'), findsNothing); + expect(find.text('AnimeJaNai models installed.'), findsOneWidget); + }); +} diff --git a/app/test/attribution_test.dart b/app/test/attribution_test.dart index 809cd0cf..2ebce35b 100644 --- a/app/test/attribution_test.dart +++ b/app/test/attribution_test.dart @@ -292,5 +292,61 @@ void main() { reason: 'About dialog names component(s) absent from NOTICES.txt.', ); }); + + // Names alone matched, so a tile could state a different licence from its + // NOTICES entry and pass (RemoveGrain's tile dropped its MIT component). + // Spellings differ ("BSL 1.0" / "Boost Software License 1.0"), so compare + // the licence families each names. + test('states the same licence as its NOTICES entry', () { + final source = File( + p.join(root, 'app', 'lib', 'views', 'about_dialog.dart'), + ).readAsStringSync(); + final tiles = RegExp(r"name:\s*'([^']+)',\s*license:\s*'([^']+)'") + .allMatches(source) + .map((m) => (m.group(1)!, m.group(2)!)) + .toList(); + final headings = { + for (final m in RegExp(r'^(\S[^\n]*?) — ([^\n]+)$', multiLine: true).allMatches(notices)) + m.group(1)!.trim(): m.group(2)!.trim(), + }; + String key(String name) => + name.split(' (').first.split(' / ').first.trim().toLowerCase(); + + final checked = []; + final disagree = []; + for (final (name, licence) in tiles) { + final entry = headings.entries.where((h) => key(h.key) == key(name)).firstOrNull; + if (entry == null) continue; // matched by its first word above + checked.add(name); + final a = _licenceFamilies(licence), b = _licenceFamilies(entry.value); + // The same licences on both sides: a qualifier ("public domain + // dedication") is not a licence family, so it may differ; a component + // under its own licence (RemoveGrain's MIT Clense) may not. + if (a.isEmpty || !(a.containsAll(b) && b.containsAll(a))) { + disagree.add('$name: About "$licence", NOTICES "${entry.value}"'); + } + } + expect(checked.length, greaterThan(40), reason: 'the pairing looks broken'); + expect(disagree, isEmpty); + }); }); } + +/// The licences a licence statement names, as families. +Set _licenceFamilies(String text) { + final t = text.toLowerCase(); + const words = { + 'lgpl': 'lgpl', 'cc by-nc-sa': 'cc-by-nc-sa', 'wtfpl': 'wtfpl', 'mit': 'mit', + 'bsd': 'bsd', 'apache': 'apache', 'isc': 'isc', 'boost': 'bsl', 'bsl': 'bsl', + 'psf': 'psf', 'python software': 'psf', 'zlib': 'zlib', 'unlicense': 'unlicense', + 'no licence': 'none stated', 'no license': 'none stated', 'no copyright': 'no copyright', + }; + final out = {for (final MapEntry(key: w, value: f) in words.entries) if (t.contains(w)) f}; + for (final m in RegExp(r'(? m.name.startsWith('real'))) { + for (final m in [...UpscaleMethod.values.where((m) => m.name.startsWith('real')), + UpscaleMethod.animeJanai]) { expect(gpuPluginSelectedBy('crop_resize', {'useIntegerUpscale': true, 'upscaleMethod': m.name}), GpuPlugin.upscale, reason: m.name); } @@ -111,6 +112,35 @@ void main() { expect(ParameterConverter.toFrameRate(unknown).method, FrameRateMethod.flowFps); }); + test('AnimeJaNai is recognised as needing its add-on only when chosen', () { + expect(animeJanaiSelectedBy('crop_resize', {'useIntegerUpscale': true, 'upscaleMethod': 'animeJanai'}), + isTrue); + expect(animeJanaiSelectedBy('crop_resize', {'useIntegerUpscale': false, 'upscaleMethod': 'animeJanai'}), + isFalse); + expect(animeJanaiSelectedBy('crop_resize', {'useIntegerUpscale': true, 'upscaleMethod': 'realEsrgan'}), + isFalse); + expect(animeJanaiSelectedBy('frame_rate', {'useIntegerUpscale': true, 'upscaleMethod': 'animeJanai'}), + isFalse); + }); + + test('the converter round-trips AnimeJaNai\'s model and Sharp settings', () { + for (final model in ['compact', 'ultraCompact', 'superUltraCompact']) { + for (final sharp in [false, true]) { + final params = CropResizeParameters( + enabled: true, + useIntegerUpscale: true, + upscaleMethod: UpscaleMethod.animeJanai, + animejanaiModel: model, + animejanaiSharp: sharp); + final back = ParameterConverter.toCropResize(ParameterConverter.fromCropResize(params)); + expect((back.upscaleMethod, back.animejanaiModel, back.animejanaiSharp), + (UpscaleMethod.animeJanai, model, sharp)); + expect(back.toJson()['upscaleMethod'], 'animeJanai', + reason: 'the wire name the worker reads'); + } + } + }); + test('the converter round-trips every upscale method and CUGAN strength', () { for (final method in UpscaleMethod.values) { for (final noise in ['none', 'conservative', 'denoise3x']) { diff --git a/app/test/integration_ai_upscale_test.dart b/app/test/integration_ai_upscale_test.dart index 6c447113..bdda6894 100644 --- a/app/test/integration_ai_upscale_test.dart +++ b/app/test/integration_ai_upscale_test.dart @@ -1,4 +1,6 @@ -// Real-ESRGAN / Real-CUGAN upscaling, encoded end to end and checked by pixels. +// Real-ESRGAN / Real-CUGAN / AnimeJaNai upscaling, encoded end to end and +// checked by pixels. AnimeJaNai's models are an add-on, which the harness +// installs from the upstream download the app pins. // // Wherever AI upscaling ships, CI gives the runner a Vulkan device (a software // one, lavapipe, on the GPU-less Linux and Windows runners) and sets @@ -25,9 +27,13 @@ // 5-20 levels). Real-ESRGAN measurably tints and saturates flat colour // (measured up to 3 levels at 2x and 11 at 4x, in fp32 too, with vs-mlrt's // own wrapper doing nothing more than this template — the model, not the -// pipeline) and CUGAN 4x drifts ~4, so those get bounds just above that; +// pipeline) and CUGAN 4x drifts ~4, so those get bounds just above that. +// AnimeJaNai keeps flat colour within half a level at 2x and 4x (measured +// 0.3 and 0.5), so it gets the strictest bound; // * the preview of a frame is the encode's frame (the same script runs); -// * a 10-bit source stays 10-bit and matches the 8-bit result. +// * a 10-bit source stays 10-bit and matches the 8-bit result; +// * AnimeJaNai picks its SD model for an SD picture (where the HD size +// setting changes nothing) and the chosen HD model above 576 lines. @Tags(['heavy']) library; @@ -54,7 +60,10 @@ const _frames = 5; String get _outDir => p.join(WorkerHarness.outputDir, 'ai_upscale'); VideoJob _job(String name, UpscaleMethod method, int factor, - {String noise = 'conservative', String? input, String pixFmt = 'yuv420p'}) => + {String noise = 'conservative', + String? input, + String pixFmt = 'yuv420p', + String animejanaiModel = 'compact'}) => VideoJob( id: const Uuid().v4(), inputPath: input ?? _input, @@ -67,6 +76,7 @@ VideoJob _job(String name, UpscaleMethod method, int factor, upscaleMethod: method, upscaleFactor: factor, cuganNoise: noise, + animejanaiModel: animejanaiModel, ), ), encodingSettings: const EncodingSettings( @@ -137,12 +147,21 @@ void main() { (UpscaleMethod.realCugan, 2, 'conservative'), (UpscaleMethod.realCugan, 2, 'denoise3x'), (UpscaleMethod.realCugan, 4, 'conservative'), + (UpscaleMethod.animeJanai, 2, ''), + (UpscaleMethod.animeJanai, 4, ''), ]; + /// The rule, plus AnimeJaNai's add-on where the method needs it. + Future ready(UpscaleMethod method) async { + if (!await requireGpuPlugin('--probe-upscale')) return false; + if (method == UpscaleMethod.animeJanai) await WorkerHarness.ensureAnimeJanai(); + return true; + } + for (final (method, factor, noise) in cases) { final name = '${method.name}_${factor}x${method == UpscaleMethod.realCugan ? '_$noise' : ''}'; test(name, () async { - if (!await requireGpuPlugin('--probe-upscale')) return; + if (!await ready(method)) return; final job = _job(name, method, factor, noise: noise); final out = await _encode(job, name); @@ -272,9 +291,10 @@ void main() { for (final (method, factor, colourTolerance) in [ (UpscaleMethod.realEsrgan, 2, 6.0), (UpscaleMethod.realEsrgan, 4, 14.0), (UpscaleMethod.realCugan, 2, 2.0), (UpscaleMethod.realCugan, 4, 6.0), + (UpscaleMethod.animeJanai, 2, 1.0), (UpscaleMethod.animeJanai, 4, 1.0), ]) { test('${method.name} ${factor}x keeps colour and position', () async { - if (!await requireGpuPlugin('--probe-upscale')) return; + if (!await ready(method)) return; final name = 'pattern_${method.name}_${factor}x'; final out = await _encode(patternJob(name, method, factor), name); final ref = await _encode( @@ -345,4 +365,43 @@ void main() { print(' 8-bit vs 10-bit: meanAbsDiff ${diff.toStringAsFixed(2)}'); expect(diff, lessThan(2.0), reason: 'the 10-bit path is not the same picture'); }, timeout: const Timeout(Duration(minutes: 30))); + + // The script chooses AnimeJaNai's model by height. On an SD picture it is + // always the SD model, so the HD size setting changes nothing; above 576 + // lines it is the chosen HD model, so two sizes give two pictures. One + // preview frame each, so a CPU Vulkan driver gets through 720p quickly. + test('AnimeJaNai uses its SD model up to 576 lines and the chosen HD model above', () async { + if (!await ready(UpscaleMethod.animeJanai)) return; + final hd = p.join(_outDir, 'source_720p.mkv'); + final made = await Process.run( + WorkerHarness.ffmpegPath, + ['-y', '-v', 'error', '-i', _input, '-frames:v', '120', '-vf', 'scale=1280:720', + '-c:v', 'ffv1', hd], + environment: WorkerHarness.ffmpegEnv, + ); + expect(made.exitCode, 0, reason: '${made.stderr}'); + + Future frame(String label, String model, {String? input}) async { + var j = _job(label, UpscaleMethod.animeJanai, 2, input: input, animejanaiModel: model); + if (input != null) { + j = VideoJob.fromJson( + {...j.toJson(), 'inputWidth': 1280, 'inputHeight': 720, 'startFrame': 100}); + } + final pv = await WorkerHarness.runPreview(j.toJson(), frame: _start + 2, label: label); + expect(pv.success, isTrue, reason: pv.errorTail); + return WorkerHarness.imageToRgb24(pv.png!, label: label); + } + + final sdCompact = await frame('ajn_sd_compact', 'compact'); + final sdTiny = await frame('ajn_sd_suc', 'superUltraCompact'); + final hdCompact = await frame('ajn_hd_compact', 'compact', input: hd); + final hdTiny = await frame('ajn_hd_suc', 'superUltraCompact', input: hd); + final sd = WorkerHarness.meanAbsDiff(sdCompact, sdTiny); + final hdDiff = WorkerHarness.meanAbsDiff(hdCompact, hdTiny); + // ignore: avoid_print + print(' AnimeJaNai Compact vs SuperUltraCompact: SD ${sd.toStringAsFixed(3)}, ' + 'HD ${hdDiff.toStringAsFixed(3)}'); + expect(sd, lessThan(0.01), reason: 'an SD picture did not use the SD model'); + expect(hdDiff, greaterThan(0.2), reason: 'a 720p picture did not use the chosen HD model'); + }, timeout: const Timeout(Duration(minutes: 30))); } diff --git a/app/test/integration_gpu_plugins_test.dart b/app/test/integration_gpu_plugins_test.dart index 5e7653ab..62ac7155 100644 --- a/app/test/integration_gpu_plugins_test.dart +++ b/app/test/integration_gpu_plugins_test.dart @@ -1,4 +1,7 @@ -// RIFE and AI upscaling (Real-ESRGAN / Real-CUGAN), through the real worker. +// RIFE and AI upscaling (Real-ESRGAN / Real-CUGAN / AnimeJaNai), through the +// real worker. AnimeJaNai's models are an add-on: the harness installs them +// from the upstream download the app's manifest pins, so this also checks that +// the pinned URL and hashes still hold. // // Both run a network through ncnn on Vulkan, and ncnn segfaults without a // Vulkan device rather than raising. So what must hold on EVERY platform is @@ -89,7 +92,9 @@ class _Feature { final String dir; final List scriptMust; final (int, int) previewSize; - const _Feature(this.plugin, this.job, this.dir, this.scriptMust, this.previewSize); + final bool needsAnimeJanai; + const _Feature(this.plugin, this.job, this.dir, this.scriptMust, this.previewSize, + {this.needsAnimeJanai = false}); } final _features = { @@ -106,6 +111,12 @@ final _features = { 'up2x-latest-denoise3x.onnx', 'core.ncnn.Model(_ai_rgb, network_path=_ai_model, fp16=True', ], (1440, 1152)), + 'AnimeJaNai': _Feature(GpuPlugin.upscale, () => _upscaleJob(UpscaleMethod.animeJanai), 'animejanai', [ + '2x_AnimeJaNai_SD_V1beta34_Compact.onnx', + '2x_AnimeJaNai_HD_V3_Compact.onnx', + '_ai_model = _ai_model[0] if _ai_rgb.height <= 576 else _ai_model[1]', + 'core.ncnn.Model(_ai_rgb, network_path=_ai_model, fp16=True', + ], (1440, 1152), needsAnimeJanai: true), }; void main() { @@ -184,6 +195,7 @@ void main() { test('where it runs, the script loads it explicitly and calls it', () async { if (!await requireGpuPlugin(f.plugin.probeFlag)) return; + if (f.needsAnimeJanai) await WorkerHarness.ensureAnimeJanai(); final script = await WorkerHarness.generateScript(f.job().toJson()); expect(script, contains('core.std.LoadPlugin(')); expect(script, contains('${p.separator}${f.dir}${p.separator}'.replaceAll(r'\', r'\\'))); @@ -197,6 +209,7 @@ void main() { test('where it runs, a real frame comes out of the GPU at the right size', () async { if (!await requireGpuPlugin(f.plugin.probeFlag)) return; + if (f.needsAnimeJanai) await WorkerHarness.ensureAnimeJanai(); final pv = await WorkerHarness.runPreview(f.job().toJson(), frame: 21, label: 'gpu_$name'); expect(pv.success, isTrue, reason: pv.errorTail); expect(_pngSize(pv.png!), f.previewSize); diff --git a/app/test/support/worker_harness.dart b/app/test/support/worker_harness.dart index efb62199..f32ab4c8 100644 --- a/app/test/support/worker_harness.dart +++ b/app/test/support/worker_harness.dart @@ -30,6 +30,7 @@ import 'package:path/path.dart' as p; // importing this file does NOT initialize the app's rhttp HTTP client, which is // why the harness does its own dart:io download below (rhttp's native bindings // don't load under the headless `flutter test` VM). +import 'package:vapourbox/services/addon_service.dart'; import 'package:vapourbox/services/dependency_manager.dart' show DepsVersionInfo, DependencyManager; @@ -108,6 +109,36 @@ class WorkerHarness { return null; } + /// Install the AnimeJaNai add-on (its models) where the worker finds it, + /// from the upstream download `assets/animejanai-addon.json` pins, with the + /// app's own extraction and hash checks — so a test of AnimeJaNai also + /// tests that the manifest's URL and hashes still hold. A debug worker finds + /// add-ons in the nearest `addons/` above it, i.e. the repo's (gitignored). + /// No-op once installed. + static Future ensureAnimeJanai() async { + final addons = Directory(_addonsDir ?? p.join(repoRoot, 'addons')); + final target = Directory(p.join(addons.path, AnimeJanaiInstaller.dirName)); + final manifest = jsonDecode( + File(p.join(repoRoot, 'app', 'assets', 'animejanai-addon.json')).readAsStringSync()) + as Map; + final installer = AnimeJanaiInstaller(manifest, platform); + if (installer.isInstalled(addons)) return target; + final archive = manifest['archive'] as Map; + stderr.writeln('[harness] downloading AnimeJaNai models from ${archive['url']}'); + final bytes = await _httpGetBytes(archive['url'] as String); + final sha = sha256.convert(bytes).toString(); + if (sha != archive['sha256']) { + throw StateError('AnimeJaNai download does not match the pinned hash: $sha'); + } + final staging = Directory('${target.path}.new'); + if (staging.existsSync()) staging.deleteSync(recursive: true); + await AnimeJanaiInstaller.extractModels( + bytes, (manifest['models'] as Map).cast(), staging); + if (target.existsSync()) target.deleteSync(recursive: true); + staging.renameSync(target.path); + return target; + } + /// Whether the whisper add-on (whisper-cli) is available. Subtitle tests that /// run the full pipeline must skip when this is false (local dev w/o add-on). static bool get whisperAvailable { diff --git a/docs/ENGINEERING_NOTES.md b/docs/ENGINEERING_NOTES.md index 0ce97625..072f032d 100644 --- a/docs/ENGINEERING_NOTES.md +++ b/docs/ENGINEERING_NOTES.md @@ -2516,6 +2516,45 @@ credits -> 720 (0.35); the VHS capture -> not found (0.91). Encode with protection: credits exact (0.0 levels), picture within 0.27 of the source; without it the credits move 62.6 levels. 8- vs 10-bit parity 0.32. +### AnimeJaNai as an add-on, and one add-on system (2026-10-25) + +**Why it is a download, not in the bundle.** AnimeJaNai's models are CC BY-NC-SA +4.0. VapourBox distributing them free is a non-commercial use, so shipping +them would be allowed. But the restriction binds every recipient: users doing +paid work (client restorations, monetised channels), and anyone who repackages +VapourBox (the GPL permits selling it). Bundling is lawful aggregation, but it +would make the deps bundle the one part of VapourBox that can't be passed on +freely, and nobody would see the terms. So: an opt-in download from upstream, +with the terms in front of the user at download time and whenever the method +is chosen. + +**Source.** The only standalone download is release `3.0.0`'s +`2x_AnimeJaNai_HD_V3_ModelsOnly.zip` (24.7 MB). The V3.1 models upstream's +README mentions exist only inside the ~170 MB full player package. The zip's +entries are dated 2024-07-30, six months after the release, so the asset has +been replaced once; it is pinned by SHA-256 and each model by its own hash. + +**Probe (M1 Pro, VSNCNN fp16).** All seven ONNX models load and run unchanged +(they are Real-ESRGAN Compact networks, the same kind as the bundled +Real-ESRGAN model). + +- Speed at 2x: Compact 8–9 fps SD / 0.9 fps 1080p; UltraCompact 14–16 / 1.7; + SuperUltraCompact 35–37 / 4. +- Colour on flat patches (direct, RGB): 0.1–0.5 levels, against Real-ESRGAN's + 3.7. Through the full YUV pipeline, AnimeJaNai is 0.3 levels off at 2x and + 0.5 at 4x, against Real-ESRGAN's 3 and 11. +- fp16 vs fp32: within 0.36/255. + +**SD vs HD.** Upstream trained the HD models on 1080p anime and warns that they +don't always suit SD. So the script picks the SD model at or below 576 lines, +per doubling. The heavy test shows the size setting has no effect on an SD +picture (difference 0.000) and does change a 720p one (0.52). + +**One add-on system.** Whisper had its own manager and dialogs. Both are now +`AddonInstaller`s behind `AddonService`, shown on Settings → Add-ons and in +the shared `AddonPrompt`. Requirements (a Whisper model needs the engine) are +installed first and removed with what needs them. + ### Testing a deps change before publishing A PR that changes `deps/` has a chicken-and-egg problem: `ci-test.yml` and diff --git a/docs/FILTER_SCHEMA.md b/docs/FILTER_SCHEMA.md index fdbd05a9..51919b48 100644 --- a/docs/FILTER_SCHEMA.md +++ b/docs/FILTER_SCHEMA.md @@ -191,6 +191,7 @@ entirely and replaced by a line telling the user more exist in advanced mode. | `hidden` | boolean | Never shown (used for `enabled`) | | `visibleWhen` | object | Conditional visibility | | `booleanLabels` | object | Names the two states of a boolean, e.g. `{"true": "Top Field First (TFF)", "false": "Bottom Field First (BFF)"}` | +| `optionLabels` | object | Names an enum's options, e.g. `{"realEsrgan": "Real-ESRGAN"}`; an option without one is title-cased, which mangles a camel-case value | There is **no `advancedOnly` on a parameter** — only on a section (below) or a method (above). diff --git a/licenses/NOTICES.txt b/licenses/NOTICES.txt index 1e316c89..b7cac9dc 100644 --- a/licenses/NOTICES.txt +++ b/licenses/NOTICES.txt @@ -254,6 +254,17 @@ Real-ESRGAN anime video model — BSD 3-Clause The trained network weights of realesr-animevideov3, at 2x and 4x. Shipped as vs-mlrt's ONNX conversions (RealESRGANv2-animevideo-xsx2/xsx4.onnx). +AnimeJaNai models — CC BY-NC-SA 4.0 (non-commercial) + By the-database (GitHub); upstream names no other copyright holder. + https://github.com/the-database/mpv-upscale-2x_animejanai + https://creativecommons.org/licenses/by-nc-sa/4.0/ + The 2x_AnimeJaNai HD V3 and SD V1 beta Real-ESRGAN Compact models, used by + the AnimeJaNai upscaler. NOT distributed with VapourBox: an optional add-on + the app downloads from upstream's own release, unmodified, only when the user + asks, after stating the licence (Settings → Add-ons, or on choosing the + upscaler). Free for non-commercial use only; anyone using them must keep to + that. + Real-CUGAN models — MIT Copyright (c) 2022 bilibili https://github.com/bilibili/ailab/tree/main/Real-CUGAN @@ -436,6 +447,15 @@ whisper.cpp — MIT downloaded optional add-on, not inside the application bundle: on macOS from the Homebrew whisper-cpp bottle, on Windows from the project's own release, on Linux from an unmodified build published on this repository's releases. + Installed and removed in Settings → Add-ons. + +Whisper models — MIT + Copyright (c) 2022 OpenAI + https://github.com/openai/whisper + The speech recognition models (small, medium, large-v3-turbo) whisper.cpp + runs, in its ggml format as published at + https://huggingface.co/ggerganov/whisper.cpp (MIT). Distributed as a + separately downloaded optional add-on, not inside the application bundle. libdvdread — GPL-2.0-or-later Copyright (c) VideoLAN and contributors @@ -511,7 +531,8 @@ MIT License ----------- (Applies to zsmooth, vszip and the zigimg library it includes, CAS, DCTFilter, BM3D, descale, the RIFE plugin, rife-ncnn-vulkan, the RIFE model and -ECCV2022-RIFE, the Real-CUGAN models, whisper.cpp, pip and the Clense filter +ECCV2022-RIFE, the Real-CUGAN models, whisper.cpp, the Whisper models, pip and +the Clense filter in vs-removegrain, each under the copyright notice given in its entry above.) diff --git a/worker/src/animejanai.rs b/worker/src/animejanai.rs new file mode 100644 index 00000000..81258865 --- /dev/null +++ b/worker/src/animejanai.rs @@ -0,0 +1,125 @@ +//! AnimeJaNai: the-database's 2x Real-ESRGAN Compact models for anime, one of +//! the Crop & Resize AI upscalers. +//! +//! Unlike Real-ESRGAN and Real-CUGAN these are not in the deps bundle. Their +//! licence is CC BY-NC-SA 4.0 — non-commercial use only — which every other +//! file VapourBox ships is free of, so they stay out of the bundle (anyone may +//! redistribute that, commercially included) and are an optional download the +//! app makes when the user picks the method, after saying so: the models come +//! from upstream's own release, pinned by hash in +//! `app/assets/animejanai-addon.json`, into `addons/animejanai/`. +//! +//! They run through the same VSNCNN plugin and script block as the bundled +//! models, so everything in `gpu_plugin.rs` (the Vulkan probe, never +//! autoloading) applies; this module only says where the files are and which +//! one a job runs. + +use std::path::{Path, PathBuf}; + +/// The models' directory under the add-ons directory. +pub const DIR_NAME: &str = "animejanai"; + +/// Every model the add-on installs. `app/assets/animejanai-addon.json` lists +/// the same files with their hashes (asserted below). +pub const MODELS: &[&str] = &[ + "2x_AnimeJaNai_HD_V3_Compact.onnx", + "2x_AnimeJaNai_HD_V3_UltraCompact.onnx", + "2x_AnimeJaNai_HD_V3_SuperUltraCompact.onnx", + "2x_AnimeJaNai_HD_V3Sharp1_Compact.onnx", + "2x_AnimeJaNai_HD_V3Sharp1_UltraCompact.onnx", + "2x_AnimeJaNai_HD_V3Sharp1_SuperUltraCompact.onnx", + "2x_AnimeJaNai_SD_V1beta34_Compact.onnx", +]; + +/// The SD model, which upstream trained separately: the HD models were +/// trained on 1080p anime and "don't always work well for SD sources". It +/// comes in one size, with no Sharp variant. +pub const SD_MODEL: &str = "2x_AnimeJaNai_SD_V1beta34_Compact.onnx"; + +/// The tallest picture the SD model is used for — PAL DVD. The script picks +/// per doubling, so a 4x upscale of a DVD runs the SD model and then an HD one, +/// the stacking upstream suggests for SD to 4K. +pub const SD_MAX_HEIGHT: u32 = 576; + +/// What the worker says when a job picks AnimeJaNai and the models are not +/// installed. The app downloads them when the method is chosen, so this is +/// reached only if they were deleted since or the job came from elsewhere. +pub const MISSING_MESSAGE: &str = "AnimeJaNai's models are not installed. They \ + are a separate download, free for non-commercial use only: choose AnimeJaNai \ + in Crop & Resize to download them."; + +/// The HD model for a size (`compact`, `ultraCompact`, `superUltraCompact`) and +/// the Sharp variant. An unknown size from a hand-edited preset is Compact. +pub fn hd_model(size: &str, sharp: bool) -> &'static str { + match (size, sharp) { + ("ultraCompact", false) => "2x_AnimeJaNai_HD_V3_UltraCompact.onnx", + ("ultraCompact", true) => "2x_AnimeJaNai_HD_V3Sharp1_UltraCompact.onnx", + ("superUltraCompact", false) => "2x_AnimeJaNai_HD_V3_SuperUltraCompact.onnx", + ("superUltraCompact", true) => "2x_AnimeJaNai_HD_V3Sharp1_SuperUltraCompact.onnx", + (_, false) => "2x_AnimeJaNai_HD_V3_Compact.onnx", + (_, true) => "2x_AnimeJaNai_HD_V3Sharp1_Compact.onnx", + } +} + +/// The installed models' directory, or the message explaining that they are +/// missing. Every model must be there: the app installs them as a set, so a +/// partial directory is a damaged install rather than a choice. +pub fn runtime(addons_dir: &Path) -> Result { + let dir = addons_dir.join(DIR_NAME); + if MODELS.iter().all(|m| dir.join(m).is_file()) { + Ok(dir) + } else { + Err(MISSING_MESSAGE.to_string()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn models_match_the_app_manifest() { + let manifest: serde_json::Value = serde_json::from_str(include_str!( + "../../app/assets/animejanai-addon.json" + )) + .unwrap(); + let mut listed: Vec<&str> = manifest["models"] + .as_object() + .unwrap() + .keys() + .map(String::as_str) + .collect(); + listed.sort(); + let mut ours = MODELS.to_vec(); + ours.sort(); + assert_eq!(listed, ours); + assert!(MODELS.contains(&SD_MODEL)); + } + + #[test] + fn every_size_and_sharpness_names_an_installed_model() { + for size in ["compact", "ultraCompact", "superUltraCompact", "huge"] { + for sharp in [false, true] { + let m = hd_model(size, sharp); + assert!(MODELS.contains(&m), "{m}"); + assert_ne!(m, SD_MODEL); + assert_eq!(m.contains("Sharp1"), sharp, "{m}"); + } + } + assert_eq!(hd_model("huge", false), hd_model("compact", false)); + } + + #[test] + fn runtime_needs_every_model() { + let tmp = std::env::temp_dir().join(format!("ajn-runtime-{}", std::process::id())); + let dir = tmp.join(DIR_NAME); + std::fs::create_dir_all(&dir).unwrap(); + for m in &MODELS[1..] { + std::fs::write(dir.join(m), b"x").unwrap(); + } + assert_eq!(runtime(&tmp), Err(MISSING_MESSAGE.to_string())); + std::fs::write(dir.join(MODELS[0]), b"x").unwrap(); + assert_eq!(runtime(&tmp), Ok(dir)); + std::fs::remove_dir_all(&tmp).unwrap(); + } +} diff --git a/worker/src/dependency_locator.rs b/worker/src/dependency_locator.rs index 2b958f09..b511fe28 100644 --- a/worker/src/dependency_locator.rs +++ b/worker/src/dependency_locator.rs @@ -862,6 +862,13 @@ impl DependencyLocator { PathBuf::from("addons") } + /// AnimeJaNai's installed model directory under the add-ons, or the + /// message explaining that the add-on is missing — what + /// `ScriptGenerator::with_animejanai` takes. + pub fn animejanai_runtime(&self) -> Result { + crate::animejanai::runtime(&self.addons_dir()) + } + /// Get the path to the libdvdread shared library. pub fn dvdread_library_path(&self) -> PathBuf { let platform_dir = self.platform_dir(); diff --git a/worker/src/lib.rs b/worker/src/lib.rs index c67fa74a..6e5d83b9 100644 --- a/worker/src/lib.rs +++ b/worker/src/lib.rs @@ -12,6 +12,7 @@ pub mod pipeline_executor; pub mod pixel_format; pub mod source_decode; pub mod progress_reporter; +pub mod animejanai; pub mod gpu_plugin; pub mod native_res; pub mod schema_script_generator; diff --git a/worker/src/main.rs b/worker/src/main.rs index fe07df17..8b7ddf36 100644 --- a/worker/src/main.rs +++ b/worker/src/main.rs @@ -26,6 +26,7 @@ mod pipeline_executor; mod pixel_format; mod source_decode; mod progress_reporter; +mod animejanai; mod gpu_plugin; mod native_res; mod script_generator; @@ -658,7 +659,11 @@ fn run_worker( .with_knlm_available(knlm_available) .with_rife(rife) .with_upscale(upscale) - .with_knlm(knlm); + .with_knlm(knlm) + .with_animejanai(match deps.as_ref() { + Some(d) if pipeline.uses_animejanai() => d.animejanai_runtime(), + _ => Err(animejanai::MISSING_MESSAGE.to_string()), + }); let script_path = script_generator .generate(&job) .with_context(|| "Failed to generate VapourSynth script")?; diff --git a/worker/src/models/crop_resize_parameters.rs b/worker/src/models/crop_resize_parameters.rs index b51b1493..129551c6 100644 --- a/worker/src/models/crop_resize_parameters.rs +++ b/worker/src/models/crop_resize_parameters.rs @@ -71,16 +71,35 @@ pub enum UpscaleMethod { RealEsrgan, /// Real-CUGAN (MIT, bilibili), on the GPU through VSNCNN. RealCugan, + /// AnimeJaNai (the-database, CC BY-NC-SA 4.0), on the GPU through VSNCNN. + /// Its models are an optional download, not in the deps bundle — see + /// `animejanai.rs`. + AnimeJanai, } impl UpscaleMethod { /// Whether this is one of the neural upscalers that need the VSNCNN plugin /// and a Vulkan device (see `gpu_plugin.rs`). pub fn is_ai(self) -> bool { - matches!(self, UpscaleMethod::RealEsrgan | UpscaleMethod::RealCugan) + matches!( + self, + UpscaleMethod::RealEsrgan | UpscaleMethod::RealCugan | UpscaleMethod::AnimeJanai + ) } } +/// One network an AI upscale runs. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AiModel { + /// A model the deps bundle ships, as a path under `vapoursynth/ncnn/models/`. + Bundled(&'static str), + /// An AnimeJaNai model pair under the add-on directory: the script runs + /// `sd` on a picture at most `animejanai::SD_MAX_HEIGHT` tall, else `hd`. + /// Decided in the script, per doubling, because only it knows the height + /// after cropping (and after a first doubling). + AnimeJanai { sd: &'static str, hd: &'static str }, +} + /// What to do with the pixel aspect ratio (SAR) on output. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] #[serde(rename_all = "camelCase")] @@ -263,6 +282,15 @@ pub struct CropResizeParameters { #[serde(default = "default_cugan_noise")] pub cugan_noise: String, + /// AnimeJaNai's HD model size: "compact" (the best), "ultraCompact" or + /// "superUltraCompact" (the fastest). + #[serde(default = "default_animejanai_model")] + pub animejanai_model: String, + + /// AnimeJaNai's Sharp variant of the HD model. + #[serde(default)] + pub animejanai_sharp: bool, + // --- nnedi3 controls for the edge-directed upscale paths --- // With EEDI3 selected these shape the nnedi3 sclip that guides it. @@ -312,6 +340,7 @@ pub struct CropResizeParameters { fn default_true() -> bool { true } fn default_upscale_factor() -> i32 { 2 } fn default_cugan_noise() -> String { "conservative".to_string() } +fn default_animejanai_model() -> String { "compact".to_string() } impl Default for CropResizeParameters { fn default() -> Self { @@ -344,6 +373,8 @@ impl Default for CropResizeParameters { upscale_method: UpscaleMethod::default(), upscale_factor: default_upscale_factor(), cugan_noise: default_cugan_noise(), + animejanai_model: default_animejanai_model(), + animejanai_sharp: false, upscale_nsize: None, upscale_neurons: None, upscale_qual: None, @@ -404,30 +435,46 @@ impl CropResizeParameters { self.enabled && self.use_integer_upscale && self.upscale_method.is_ai() } + /// Whether this pass upscales with AnimeJaNai, whose models must also be + /// installed as an add-on. + pub fn uses_animejanai(&self) -> bool { + self.uses_ai_upscale() && self.upscale_method == UpscaleMethod::AnimeJanai + } + /// The models an AI upscale runs, in order, as paths under /// `vapoursynth/ncnn/models/`. Factors follow the edge-directed path's /// rule (3x behaves as 2x). Real-ESRGAN has a 4x model; Real-CUGAN's /// bundled models are 2x only, so 4x runs it twice, denoising on the first /// pass only — a second denoise pass would smooth the result twice. - pub fn ai_upscale_models(&self) -> Vec<&'static str> { + /// AnimeJaNai is 2x only, so 4x is two doublings; the script picks SD or + /// HD for each, so a DVD runs the SD model and then an HD one. + pub fn ai_upscale_models(&self) -> Vec { let four = self.upscale_factor >= 4; + let bundled = |names: Vec<&'static str>| names.into_iter().map(AiModel::Bundled).collect(); match self.upscale_method { - UpscaleMethod::RealEsrgan => vec![if four { + UpscaleMethod::AnimeJanai => { + let pair = AiModel::AnimeJanai { + sd: crate::animejanai::SD_MODEL, + hd: crate::animejanai::hd_model(&self.animejanai_model, self.animejanai_sharp), + }; + vec![pair; if four { 2 } else { 1 }] + } + UpscaleMethod::RealEsrgan => bundled(vec![if four { "RealESRGANv2/RealESRGANv2-animevideo-xsx4.onnx" } else { "RealESRGANv2/RealESRGANv2-animevideo-xsx2.onnx" - }], + }]), UpscaleMethod::RealCugan => { let first = match self.cugan_noise.as_str() { "none" => "cugan/up2x-latest-no-denoise.onnx", "denoise3x" => "cugan/up2x-latest-denoise3x.onnx", _ => "cugan/up2x-latest-conservative.onnx", }; - if four { + bundled(if four { vec![first, "cugan/up2x-latest-no-denoise.onnx"] } else { vec![first] - } + }) } _ => Vec::new(), } @@ -734,24 +781,26 @@ mod tests { cugan_noise: noise.into(), ..Default::default() }; + let b = |names: &[&'static str]| names.iter().map(|n| AiModel::Bundled(n)).collect::>(); assert_eq!(p(UpscaleMethod::RealEsrgan, 2, "").ai_upscale_models(), - ["RealESRGANv2/RealESRGANv2-animevideo-xsx2.onnx"]); + b(&["RealESRGANv2/RealESRGANv2-animevideo-xsx2.onnx"])); assert_eq!(p(UpscaleMethod::RealEsrgan, 3, "").ai_upscale_models(), - ["RealESRGANv2/RealESRGANv2-animevideo-xsx2.onnx"]); + b(&["RealESRGANv2/RealESRGANv2-animevideo-xsx2.onnx"])); assert_eq!(p(UpscaleMethod::RealEsrgan, 4, "").ai_upscale_models(), - ["RealESRGANv2/RealESRGANv2-animevideo-xsx4.onnx"]); + b(&["RealESRGANv2/RealESRGANv2-animevideo-xsx4.onnx"])); assert_eq!(p(UpscaleMethod::RealCugan, 2, "denoise3x").ai_upscale_models(), - ["cugan/up2x-latest-denoise3x.onnx"]); + b(&["cugan/up2x-latest-denoise3x.onnx"])); assert_eq!(p(UpscaleMethod::RealCugan, 4, "none").ai_upscale_models(), - ["cugan/up2x-latest-no-denoise.onnx", "cugan/up2x-latest-no-denoise.onnx"]); + b(&["cugan/up2x-latest-no-denoise.onnx", "cugan/up2x-latest-no-denoise.onnx"])); // An unknown strength from a hand-edited preset is the default. assert_eq!(p(UpscaleMethod::RealCugan, 2, "max").ai_upscale_models(), - ["cugan/up2x-latest-conservative.onnx"]); + b(&["cugan/up2x-latest-conservative.onnx"])); // Every model named here is one the deps bundle ships. for m in [UpscaleMethod::RealEsrgan, UpscaleMethod::RealCugan] { for f in [2, 4] { for n in ["none", "conservative", "denoise3x"] { for model in p(m, f, n).ai_upscale_models() { + let AiModel::Bundled(model) = model else { panic!("{model:?}") }; assert!(crate::gpu_plugin::UPSCALE_MODELS.contains(&model), "{model}"); } } @@ -759,5 +808,32 @@ mod tests { } assert!(!p(UpscaleMethod::Nnedi3Rpow2, 2, "").uses_ai_upscale()); assert!(p(UpscaleMethod::RealCugan, 2, "").uses_ai_upscale()); + assert!(!p(UpscaleMethod::RealCugan, 2, "").uses_animejanai()); + } + + #[test] + fn test_animejanai_models_pair_sd_with_the_chosen_hd_model() { + let p = |factor, size: &str, sharp| CropResizeParameters { + enabled: true, + use_integer_upscale: true, + upscale_method: UpscaleMethod::AnimeJanai, + upscale_factor: factor, + animejanai_model: size.into(), + animejanai_sharp: sharp, + ..Default::default() + }; + let pair = |hd| AiModel::AnimeJanai { sd: crate::animejanai::SD_MODEL, hd }; + assert_eq!(p(2, "compact", false).ai_upscale_models(), + [pair("2x_AnimeJaNai_HD_V3_Compact.onnx")]); + assert_eq!(p(3, "superUltraCompact", true).ai_upscale_models(), + [pair("2x_AnimeJaNai_HD_V3Sharp1_SuperUltraCompact.onnx")]); + // 2x only: 4x is two doublings, each choosing SD or HD. + assert_eq!(p(4, "ultraCompact", false).ai_upscale_models(), + [pair("2x_AnimeJaNai_HD_V3_UltraCompact.onnx"); 2]); + assert!(p(2, "compact", false).uses_animejanai()); + assert!(p(2, "compact", false).uses_ai_upscale()); + // Defaults: Compact, not sharp. + let d = CropResizeParameters::default(); + assert_eq!((d.animejanai_model.as_str(), d.animejanai_sharp), ("compact", false)); } } diff --git a/worker/src/models/processing_pipeline.rs b/worker/src/models/processing_pipeline.rs index c07371db..7357cd3c 100644 --- a/worker/src/models/processing_pipeline.rs +++ b/worker/src/models/processing_pipeline.rs @@ -489,6 +489,12 @@ impl ProcessingPipeline { self.crop_resize.uses_ai_upscale() } + /// Whether that model is AnimeJaNai, whose models are an add-on that must + /// also be installed. + pub fn uses_animejanai(&self) -> bool { + self.crop_resize.uses_animejanai() + } + /// Whether this pipeline denoises with KNLMeans, which needs a working /// OpenCL device before its script can be generated. pub fn uses_knlm(&self) -> bool { diff --git a/worker/src/pipeline_executor.rs b/worker/src/pipeline_executor.rs index 97ed5bbb..366085f2 100644 --- a/worker/src/pipeline_executor.rs +++ b/worker/src/pipeline_executor.rs @@ -1264,7 +1264,8 @@ impl PipelineExecutor { self.deps.knlm_runtime() } else { Err("KNLMeans is not used by this job.".into()) - }); + }) + .with_animejanai(self.deps.animejanai_runtime()); let (fps_num, fps_den) = script_generator.frame_rate_to_rational(frame_rate); let preview_params = PreviewParams { diff --git a/worker/src/script_generator.rs b/worker/src/script_generator.rs index 4399806f..f902693a 100644 --- a/worker/src/script_generator.rs +++ b/worker/src/script_generator.rs @@ -11,7 +11,7 @@ use anyhow::{Context, Result}; use crate::gpu_plugin::{py_str, GpuPluginFiles}; use crate::models::{ AntiAliasMethod, GrainMethod, - VideoJob, ProcessingPipeline, NoiseReductionMethod, UpscaleMethod, CCD_REFERENCE_HEIGHT, + VideoJob, ProcessingPipeline, NoiseReductionMethod, UpscaleMethod, AiModel, CCD_REFERENCE_HEIGHT, ChromaDenoiseMethod, FrameRateMethod, SpotLessMethod, DeflickerMethod, EdgeRepairParameters, parse_ratio, DehaloMethod, DeblockMethod, SharpenMethod, DeinterlaceMethod, @@ -77,6 +77,11 @@ pub struct ScriptGenerator { /// different question: whether QTGMC's KNLMeansCL call, which havsfunc makes /// with the plugin's default device, works.) knlm: std::result::Result, + /// AnimeJaNai's installed model directory, or why it is missing. The + /// models are an add-on (see `animejanai.rs`), so this is checked in + /// addition to `upscale`. Defaults to "not checked"; callers set it from + /// `DependencyLocator::animejanai_runtime()`. + animejanai: std::result::Result, } /// Parameters for preview script generation. @@ -111,6 +116,7 @@ impl ScriptGenerator { rife: Err("RIFE was not checked for this job.".to_string()), upscale: Err("AI upscaling was not checked for this job.".to_string()), knlm: Err("KNLMeansCL was not checked for this job.".to_string()), + animejanai: Err("AnimeJaNai was not checked for this job.".to_string()), }) } @@ -141,6 +147,12 @@ impl ScriptGenerator { self } + /// Set AnimeJaNai's model directory, or why it is missing (see `self.animejanai`). + pub fn with_animejanai(mut self, animejanai: std::result::Result) -> Self { + self.animejanai = animejanai; + self + } + /// Set the device KNLMeans runs on, or why it cannot run (see `self.knlm`). pub fn with_knlm(mut self, knlm: std::result::Result) -> Self { self.knlm = knlm; @@ -160,6 +172,11 @@ impl ScriptGenerator { anyhow::bail!("{message}"); } } + if pipeline.uses_animejanai() { + if let Err(message) = &self.animejanai { + anyhow::bail!("{message}"); + } + } if pipeline.uses_knlm() { if let Err(message) = &self.knlm { anyhow::bail!("{message}"); @@ -2050,7 +2067,14 @@ impl ScriptGenerator { resize .ai_upscale_models() .iter() - .map(|m| py_str(&f.model_dir.join(m))) + .map(|m| match *m { + AiModel::Bundled(name) => py_str(&f.model_dir.join(name)), + // A pair the script chooses from by height. + AiModel::AnimeJanai { sd, hd } => { + let dir = self.animejanai.as_deref().unwrap_or(Path::new("")); + format!("({}, {})", py_str(&dir.join(sd)), py_str(&dir.join(hd))) + } + }) .collect::>() .join(", "), ), @@ -2059,6 +2083,10 @@ impl ScriptGenerator { script = script.replace("{{AI_PLUGIN}}", &plugin); script = script.replace("{{AI_PRELOAD}}", &preload); script = script.replace("{{AI_MODELS}}", &models); + script = script.replace( + "{{AI_SD_MAX_HEIGHT}}", + &crate::animejanai::SD_MAX_HEIGHT.to_string(), + ); let color = job.color_metadata(); script = script.replace("{{AI_MATRIX}}", color.zimg_matrix(job.input_height)); script = script.replace( @@ -2095,7 +2123,10 @@ impl ScriptGenerator { script = process_optional_int("UPSCALE_NRAD", resize.upscale_nrad, script); script = process_optional_int("UPSCALE_MDIS", resize.upscale_mdis, script); } - UpscaleMethod::Spline36 | UpscaleMethod::RealEsrgan | UpscaleMethod::RealCugan => { + UpscaleMethod::Spline36 + | UpscaleMethod::RealEsrgan + | UpscaleMethod::RealCugan + | UpscaleMethod::AnimeJanai => { script = remove_block("{{#UPSCALE_NNEDI3}}", "{{/UPSCALE_NNEDI3}}", script); script = remove_block("{{#UPSCALE_EEDI3}}", "{{/UPSCALE_EEDI3}}", script); } diff --git a/worker/templates/pipeline_template.vpy b/worker/templates/pipeline_template.vpy index 620e5145..05f97482 100644 --- a/worker/templates/pipeline_template.vpy +++ b/worker/templates/pipeline_template.vpy @@ -2023,6 +2023,10 @@ if _ai_pad_w or _ai_pad_h: src_width=_ai_rgb.width + _ai_pad_w, src_height=_ai_rgb.height + _ai_pad_h) _ai_in_w = _ai_rgb.width for _ai_model in [{{AI_MODELS}}]: + if isinstance(_ai_model, tuple): + # AnimeJaNai: its SD model for a picture at most {{AI_SD_MAX_HEIGHT}} tall, else + # the HD one (trained on 1080p anime), chosen per doubling. + _ai_model = _ai_model[0] if _ai_rgb.height <= {{AI_SD_MAX_HEIGHT}} else _ai_model[1] _ai_tiles = {} if _ai_rgb.width * _ai_rgb.height <= 720 * 576 else dict(tilesize=[512, 512], overlap=[16, 16]) _ai_rgb = core.ncnn.Model(_ai_rgb, network_path=_ai_model, fp16=True, num_streams=2, **_ai_tiles) _ai_scale = _ai_rgb.width // _ai_in_w diff --git a/worker/templates/preview_template.vpy b/worker/templates/preview_template.vpy index 6ac71b10..597e6fbc 100644 --- a/worker/templates/preview_template.vpy +++ b/worker/templates/preview_template.vpy @@ -1955,6 +1955,10 @@ if _ai_pad_w or _ai_pad_h: src_width=_ai_rgb.width + _ai_pad_w, src_height=_ai_rgb.height + _ai_pad_h) _ai_in_w = _ai_rgb.width for _ai_model in [{{AI_MODELS}}]: + if isinstance(_ai_model, tuple): + # AnimeJaNai: its SD model for a picture at most {{AI_SD_MAX_HEIGHT}} tall, else + # the HD one (trained on 1080p anime), chosen per doubling. + _ai_model = _ai_model[0] if _ai_rgb.height <= {{AI_SD_MAX_HEIGHT}} else _ai_model[1] _ai_tiles = {} if _ai_rgb.width * _ai_rgb.height <= 720 * 576 else dict(tilesize=[512, 512], overlap=[16, 16]) _ai_rgb = core.ncnn.Model(_ai_rgb, network_path=_ai_model, fp16=True, num_streams=2, **_ai_tiles) _ai_scale = _ai_rgb.width // _ai_in_w diff --git a/worker/tests/filter_integration_test.rs b/worker/tests/filter_integration_test.rs index 89f682b8..65f9f28b 100644 --- a/worker/tests/filter_integration_test.rs +++ b/worker/tests/filter_integration_test.rs @@ -6553,3 +6553,55 @@ fn test_177_descale_order_and_off() { }); assert!(!script_text(&off).contains("_ds_"), "descale emitted with the pass off"); } + +/// AnimeJaNai: the same VSNCNN block, with each doubling an (SD, HD) pair from +/// the add-on directory that the script chooses between by height; the chosen +/// size and Sharp variant pick the HD model, in both scripts. +#[test] +fn test_178_animejanai_emits_sd_hd_pairs_from_the_addon() { + create_output_dir(); + let files = upscale_files(); + let dir = std::path::PathBuf::from("/addons/animejanai"); + let py = |m: &str| vapourbox_worker::gpu_plugin::py_str(&dir.join(m)); + let generator = ScriptGenerator::new() + .unwrap() + .with_upscale(Ok(files.clone())) + .with_animejanai(Ok(dir.clone())); + let sd = "2x_AnimeJaNai_SD_V1beta34_Compact.onnx"; + let mut job = ai_upscale_job("test_178_animejanai", UpscaleMethod::AnimeJanai, 2, ""); + job.processing_pipeline.as_mut().unwrap().crop_resize.animejanai_model = "ultraCompact".into(); + job.processing_pipeline.as_mut().unwrap().crop_resize.animejanai_sharp = true; + let (encode, preview) = both_scripts_with(&generator, &job).unwrap(); + let pair = format!("({}, {})", py(sd), py("2x_AnimeJaNai_HD_V3Sharp1_UltraCompact.onnx")); + for (name, script) in [("encode", &encode), ("preview", &preview)] { + assert!(script.contains(&format!("for _ai_model in [{pair}]:")), "{name}: wrong models"); + assert!(script.contains("_ai_model = _ai_model[0] if _ai_rgb.height <= 576 else _ai_model[1]"), + "{name}: no SD/HD choice"); + assert!(script.contains(&format!("core.std.LoadPlugin({})", + vapourbox_worker::gpu_plugin::py_str(&files.plugin))), "{name}: VSNCNN not loaded"); + assert!(!script.contains("{{AI_"), "{name}: unsubstituted placeholder"); + } + // 4x: two doublings, each its own choice. + job.processing_pipeline.as_mut().unwrap().crop_resize.upscale_factor = 4; + job.processing_pipeline.as_mut().unwrap().crop_resize.animejanai_sharp = false; + let four = script_text_with(&generator, &job); + let pair = format!("({}, {})", py(sd), py("2x_AnimeJaNai_HD_V3_UltraCompact.onnx")); + assert!(four.contains(&format!("for _ai_model in [{pair}, {pair}]:")), "4x models"); +} + +/// Without the add-on an AnimeJaNai job is refused with the message saying +/// where to get it, even where VSNCNN runs; the bundled models are unaffected. +#[test] +fn test_179_animejanai_is_refused_without_its_models() { + create_output_dir(); + let generator = ScriptGenerator::new() + .unwrap() + .with_upscale(Ok(upscale_files())) + .with_animejanai(Err(vapourbox_worker::animejanai::MISSING_MESSAGE.into())); + let job = ai_upscale_job("test_179_refused", UpscaleMethod::AnimeJanai, 2, ""); + let err = both_scripts_with(&generator, &job).unwrap_err().to_string(); + assert!(err.contains("AnimeJaNai's models are not installed"), "{err}"); + assert!(err.contains("non-commercial"), "{err}"); + let esrgan = ai_upscale_job("test_179_esrgan", UpscaleMethod::RealEsrgan, 2, ""); + both_scripts_with(&generator, &esrgan).expect("Real-ESRGAN needs no add-on"); +} From 6f129f2c1021c4895757c6e46b6840a0c0a28b56 Mon Sep 17 00:00:00 2001 From: Stuart Cameron Date: Sun, 25 Oct 2026 13:27:57 +1100 Subject: [PATCH 03/15] fix(deps): VSNCNN arm64 deployment target; make set -e work in plugin blocks - macOS arm64 pins no deployment target, so VSNCNN's direct clang++ call got an empty -mmacosx-version-min= and failed. It now targets 15.0, the arm64 bundle's floor. - The failure went unnoticed because bash ignores set -e in a subshell used as an `if` condition: `if ( set -e ... ); then` reported success because the last command succeeded. The four macOS blocks and two Linux blocks that used that form now run the subshell on its own and check its status. - The vapoursynth shim is now written by a function, and restored near the end if missing. A rerun that reinstalled Python but skipped VapourSynth had left no shim, so VSScript could not configure itself. Co-Authored-By: Claude Opus 5.5 --- Scripts/download-deps-linux.sh | 24 ++++++++-- Scripts/download-deps-macos.sh | 88 ++++++++++++++++++++++++++-------- 2 files changed, 88 insertions(+), 24 deletions(-) diff --git a/Scripts/download-deps-linux.sh b/Scripts/download-deps-linux.sh index 06a90282..87ec312c 100755 --- a/Scripts/download-deps-linux.sh +++ b/Scripts/download-deps-linux.sh @@ -1663,7 +1663,12 @@ elif [ "$FORCE" = false ] && [ -f "$RIFE_DIR/librife.so" ] \ echo " RIFE already exists, skipping" else rm -rf "$RIFE_DIR" - if ( + # Run on its own, not as `if ( set -e ... ); then`: bash ignores set -e in + # anything that is an if/&&/|| condition, subshells included, so that form + # reported success whenever the last command did (VSNCNN's failed compile + # printed "Installed" in CI on 2026-10-25). + set +e + ( set -e mkdir -p "$RIFE_DIR/models/$RIFE_MODEL_NAME" cd "$BUILD_DIR" @@ -1679,7 +1684,10 @@ else echo "$RIFE_MODEL_BIN_SHA256 flownet.bin" | sha256sum -c - echo "$RIFE_MODEL_PARAM_SHA256 flownet.param" | sha256sum -c - mv flownet.bin flownet.param "$RIFE_DIR/models/$RIFE_MODEL_NAME/" - ); then + ) + _block_status=$? + set -e + if [ "$_block_status" -eq 0 ]; then echo " Installed RIFE -> vapoursynth/rife/" BUILT_PLUGINS+=("rife") else @@ -1737,7 +1745,12 @@ elif [ "$FORCE" = false ] && [ -f "$NCNN_DIR/libvsncnn.so" ] \ echo " VSNCNN already exists, skipping" else rm -rf "$NCNN_DIR" - if ( + # Run on its own, not as `if ( set -e ... ); then`: bash ignores set -e in + # anything that is an if/&&/|| condition, subshells included, so that form + # reported success whenever the last command did (VSNCNN's failed compile + # printed "Installed" in CI on 2026-10-25). + set +e + ( set -e W="$BUILD_DIR/vsncnn" rm -rf "$W" && mkdir -p "$W" "$NCNN_DIR/models" @@ -1794,7 +1807,10 @@ else mv "models/$rel" "$NCNN_DIR/models/$rel" done cd "$BUILD_DIR" && rm -rf "$W" - ); then + ) + _block_status=$? + set -e + if [ "$_block_status" -eq 0 ]; then echo " Installed VSNCNN -> vapoursynth/ncnn/" BUILT_PLUGINS+=("vsncnn") else diff --git a/Scripts/download-deps-macos.sh b/Scripts/download-deps-macos.sh index 788b21d0..90b40a69 100755 --- a/Scripts/download-deps-macos.sh +++ b/Scripts/download-deps-macos.sh @@ -127,6 +127,21 @@ if [ -n "$TIER" ] && [ "$FORCE" = false ] && [ -f "$DEPS_DIR/version.json" ]; th fi fi PYTHON_DIR="$DEPS_DIR/python" + +# vsscript self-configures by shelling out to a `vapoursynth` executable, and a +# from-source build creates no console script, so provide one. It lives in the +# Python tree, which a rerun can reinstall while skipping VapourSynth, so it is +# written by the VapourSynth step and again (if missing) near the end. +write_vapoursynth_shim() { + cat > "$PYTHON_DIR/bin/vapoursynth" << 'SHIM_EOF' +#!/bin/bash +DEPS_ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +export PYTHONHOME="$DEPS_ROOT/python" +export PYTHONPATH="$DEPS_ROOT:$DEPS_ROOT/python-packages:${PYTHONPATH:-}" +exec "$DEPS_ROOT/python/bin/python3" -m vapoursynth "$@" +SHIM_EOF + chmod +x "$PYTHON_DIR/bin/vapoursynth" +} PYTHON_PACKAGES_DIR="$DEPS_DIR/python-packages" BUILD_DIR="/tmp/vapourbox-build-$$" @@ -542,16 +557,7 @@ CYEOF "$DEPS_DIR/vapoursynth/" 2>/dev/null || \ cp "$VS_BUILD_DIR/src/py/"*.py "$DEPS_DIR/vapoursynth/" - # vsscript self-configures by shelling out to a `vapoursynth` executable. - # A from-source build creates no console script, so provide one. - cat > "$PYTHON_DIR/bin/vapoursynth" << 'SHIM_EOF' -#!/bin/bash -DEPS_ROOT="$(cd "$(dirname "$0")/../.." && pwd)" -export PYTHONHOME="$DEPS_ROOT/python" -export PYTHONPATH="$DEPS_ROOT:$DEPS_ROOT/python-packages:${PYTHONPATH:-}" -exec "$DEPS_ROOT/python/bin/python3" -m vapoursynth "$@" -SHIM_EOF - chmod +x "$PYTHON_DIR/bin/vapoursynth" + write_vapoursynth_shim # Copy zimg from Homebrew (will fix paths to be relative) cp "$BREW_PREFIX/lib/libzimg.2.dylib" "$DEPS_DIR/vapoursynth/libzimg.dylib" @@ -2157,7 +2163,12 @@ if [ "$FORCE" = false ] && [ -f "$DESCALE_OUT" ]; then echo " descale already exists, skipping" else rm -f "$DESCALE_OUT" - if ( + # Run on its own, not as `if ( set -e ... ); then`: bash ignores set -e in + # anything that is an if/&&/|| condition, subshells included, so that form + # reported success whenever the last command did (VSNCNN's failed compile + # printed "Installed" in CI on 2026-10-25). + set +e + ( set -e cd "$BUILD_DIR" rm -rf descale-whl descale.whl @@ -2176,7 +2187,10 @@ PYEOF2 install_name_tool -id "@loader_path/libdescale.dylib" "$DESCALE_OUT" codesign -s - -f "$DESCALE_OUT" rm -rf descale-whl descale.whl - ); then + ) + _block_status=$? + set -e + if [ "$_block_status" -eq 0 ]; then echo " Installed descale -> libdescale.dylib" BUILT_PLUGINS+=("descale") else @@ -2204,7 +2218,12 @@ elif [ "$FORCE" = false ] && [ -f "$MOLTENVK_DIR/libMoltenVK.dylib" ]; then echo " MoltenVK already exists, skipping" else rm -rf "$MOLTENVK_DIR" - if ( + # Run on its own, not as `if ( set -e ... ); then`: bash ignores set -e in + # anything that is an if/&&/|| condition, subshells included, so that form + # reported success whenever the last command did (VSNCNN's failed compile + # printed "Installed" in CI on 2026-10-25). + set +e + ( set -e mkdir -p "$MOLTENVK_DIR" cd "$BUILD_DIR" @@ -2220,7 +2239,10 @@ else rm -rf moltenvk MoltenVK-macos.tar chmod u+w "$MOLTENVK_DIR/libMoltenVK.dylib" codesign -s - -f "$MOLTENVK_DIR/libMoltenVK.dylib" - ); then + ) + _block_status=$? + set -e + if [ "$_block_status" -eq 0 ]; then echo " Installed MoltenVK -> vapoursynth/moltenvk/" BUILT_PLUGINS+=("moltenvk") else @@ -2261,7 +2283,12 @@ elif [ "$FORCE" = false ] && [ -f "$RIFE_DIR/librife.dylib" ] \ echo " RIFE already exists, skipping" else rm -rf "$RIFE_DIR" - if ( + # Run on its own, not as `if ( set -e ... ); then`: bash ignores set -e in + # anything that is an if/&&/|| condition, subshells included, so that form + # reported success whenever the last command did (VSNCNN's failed compile + # printed "Installed" in CI on 2026-10-25). + set +e + ( set -e mkdir -p "$RIFE_DIR/models/$RIFE_MODEL_NAME" cd "$BUILD_DIR" @@ -2278,7 +2305,10 @@ else mv flownet.bin flownet.param "$RIFE_DIR/models/$RIFE_MODEL_NAME/" chmod u+w "$RIFE_DIR/librife.dylib" codesign -s - -f "$RIFE_DIR/librife.dylib" - ); then + ) + _block_status=$? + set -e + if [ "$_block_status" -eq 0 ]; then echo " Installed RIFE -> vapoursynth/rife/" BUILT_PLUGINS+=("rife") else @@ -2335,12 +2365,20 @@ elif [ "$FORCE" = false ] && [ -f "$NCNN_DIR/libvsncnn.dylib" ] \ echo " VSNCNN already exists, skipping" else rm -rf "$NCNN_DIR" - if ( + # Run on its own, not as `if ( set -e ... ); then`: bash ignores set -e in + # anything that is an if/&&/|| condition, subshells included, so that form + # reported success whenever the last command did (VSNCNN's failed compile + # printed "Installed" in CI on 2026-10-25). + set +e + ( set -e W="$BUILD_DIR/vsncnn" rm -rf "$W" && mkdir -p "$W" "$NCNN_DIR/models" cd "$W" - export MACOSX_DEPLOYMENT_TARGET="$MACOS_MIN_VERSION" + # arm64 pins no target for the rest of the script; this one compiles + # directly with clang, so it needs one: the arm64 bundle's 15.0. + VSNCNN_MIN="${MACOS_MIN_VERSION:-15.0}" + export MACOSX_DEPLOYMENT_TARGET="$VSNCNN_MIN" # protobuf, static. curl -fsSL "https://github.com/protocolbuffers/protobuf/archive/refs/tags/$PROTOBUF_TAG.tar.gz" | tar xz cmake -S "protobuf-${PROTOBUF_TAG#v}/cmake" -B pb-build -G Ninja -DCMAKE_BUILD_TYPE=Release \ @@ -2374,7 +2412,7 @@ else # VSNCNN itself. git clone -q --depth 1 --branch "$VSMLRT_TAG" https://github.com/AmusementClub/vs-mlrt.git vs-mlrt echo "#define VERSION \"$VSMLRT_TAG\"" > vs-mlrt/vsncnn/config.h - clang++ -std=c++20 -O2 -fPIC -shared -arch arm64 -mmacosx-version-min="$MACOS_MIN_VERSION" \ + clang++ -std=c++20 -O2 -fPIC -shared -arch arm64 -mmacosx-version-min="$VSNCNN_MIN" \ -fvisibility=hidden \ vs-mlrt/vsncnn/vs_ncnn.cpp vs-mlrt/vsncnn/onnx2ncnn.cpp vs-mlrt/common/onnx_utils.cpp \ -Ivs-mlrt/vsncnn -I"$W/vsapi3" -I"$VS_INC_DIR" -I"$N/ncnn.framework/Headers" -I"$N/ncnn.framework/Headers/ncnn" \ @@ -2404,7 +2442,10 @@ else done rm -f models.7z cd "$BUILD_DIR" && rm -rf "$W" - ); then + ) + _block_status=$? + set -e + if [ "$_block_status" -eq 0 ]; then echo " Installed VSNCNN -> vapoursynth/ncnn/" BUILT_PLUGINS+=("vsncnn") else @@ -2917,6 +2958,13 @@ fi # is built on/for current macOS, so skip the check there to avoid spurious noise. if [ "$ARCH" = "x86_64" ]; then echo "" +# A rerun that reinstalled Python but skipped VapourSynth would otherwise leave +# no shim, and every script would fail to start VSScript. +if [ ! -x "$PYTHON_DIR/bin/vapoursynth" ]; then + write_vapoursynth_shim + echo "Restored the vapoursynth shim in python/bin" +fi + echo "Minimum-OS (minos) of every bundled Mach-O (target $MACOS_MIN_VERSION, issue #39):" # Print the macOS deployment target baked into a Mach-O, reading whichever load # command it carries: LC_BUILD_VERSION (newer toolchains, "minos X.Y") or From 619b233c4e7ede7e3149e18a6ef2087973cbc982 Mon Sep 17 00:00:00 2001 From: Stuart Cameron Date: Sun, 25 Oct 2026 13:45:45 +1100 Subject: [PATCH 04/15] docs(readme): anime overview, AI upscaling and Descale details Co-Authored-By: Claude Opus 5.5 --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 8aea28db..10d6f548 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,8 @@ VapourBox runs [VapourSynth](https://www.vapoursynth.com/), QTGMC and FFmpeg — **Cleanup, when the source needs it.** Twenty-one optional filters covering tape noise and smeared color, dust and scratches on scanned film, the dirty rows at the edge of a capture, ghosting from an aerial, the brightness flicker of scanned cine film, blocking from a heavily compressed disc or recorder, jagged diagonal edges, camera shake, and halos, banding, color balance and detail lost in dark footage. All off by default, and the presets turn on what a given kind of source actually needs. +**Anime.** Line art has needs of its own, and VapourBox covers the usual anime toolkit. **Descale** undoes the upscale most HD anime went through in production: its Detect button finds the resolution the show was drawn at and the scaler the studio used, then rebuilds the picture without the softness and halos, leaving credits and on-screen text alone. **BM3D** cleans flat colour without smearing lines. **AI upscaling** redraws line art at 2x or 4x on the graphics card with Real-ESRGAN, Real-CUGAN or AnimeJaNai. **RIFE** interpolates frames for frame-rate conversion. The Anime DVD preset handles inverse telecine, halos, banding and rainbowing on SD discs. + **Subtitles.** Speech is transcribed with Whisper AI to a separate `.srt`, embedded in the output, or both. ## How it works @@ -216,6 +218,8 @@ Shortcuts without ⌘/Ctrl are ignored while you are typing in a text box, so th - **Higher-precision deinterlacing**, both opt-in: interlaced 4:2:0 stores chroma per field, so the pass can upsample to 4:2:2 first; separately it can run at 16-bit, avoiding rounding accumulated across QTGMC's many internal steps. Both are off by default and each states what it costs. - **Soft telecine** pulldown flags can be stripped without re-encoding fields. - **Edge-directed upscaling** — NNEDI3 or EEDI3 integer doubling with full neuron, quality and prescreener control, plus seven resampling kernels with per-kernel tuning. +- **AI upscaling** at 2x or 4x on the GPU: Real-ESRGAN (sharpest, fastest), Real-CUGAN (gentler, with three denoise strengths) or AnimeJaNai (three model sizes trading detail for speed, and a Sharp variant; an optional non-commercial download). HD frames are processed in tiles, so 1080p fits in a laptop GPU's memory. +- **Descale tuning** — the studio's kernel (bilinear, bicubic with its b/c, Lanczos with its taps, Spline16/36/64) and native height, which Detect fills in, plus an explicit native width, edge handling, and how far a pixel may miss before it counts as credits or text and is kept. From 08d63e1bd1a2b0657ff1394bb87d874ad7e065f2 Mon Sep 17 00:00:00 2001 From: Stuart Cameron Date: Sun, 25 Oct 2026 14:13:48 +1100 Subject: [PATCH 05/15] fix(crop-resize): upscale descriptions cover every method, not just NNEDI3 Co-Authored-By: Claude Opus 5.5 --- app/assets/filters/core/crop_resize.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/assets/filters/core/crop_resize.json b/app/assets/filters/core/crop_resize.json index 3906c393..342c43df 100644 --- a/app/assets/filters/core/crop_resize.json +++ b/app/assets/filters/core/crop_resize.json @@ -4,7 +4,7 @@ "version": "1.3.0", "name": "Crop & Resize", "description": "Crop borders, resize or upscale, and add bars to a target size", - "longDescription": "Trims unwanted borders, changes the output resolution, and adds bars out to a target frame size. Cropping happens first, then resizing, then the bars.\n\nUse crop to cut the head-switching noise along the bottom of VHS captures and the black overscan edges of broadcast material — otherwise the encoder spends bitrate on them. Use resize for a target resolution, or the NNEDI3 upscaler for a much better 2x/4x enlargement than a plain kernel. Keep crop values even so they stay aligned with chroma subsampling.\n\nUse Add Borders to bring a cropped picture back to an exact frame size for authoring (720x576 for PAL DVD, 720x480 for NTSC) without rescaling it: the picture is centred and the rest filled with bars.", + "longDescription": "Trims unwanted borders, changes the output resolution, and adds bars out to a target frame size. Cropping happens first, then resizing, then the bars.\n\nUse crop to cut the head-switching noise along the bottom of VHS captures and the black overscan edges of broadcast material — otherwise the encoder spends bitrate on them. Use resize for a target resolution, or Upscale for a much better 2x/4x enlargement than a plain kernel: NNEDI3 or EEDI3 on any machine, or an AI upscaler trained on anime (Real-ESRGAN, Real-CUGAN, AnimeJaNai) on the graphics card. Keep crop values even so they stay aligned with chroma subsampling.\n\nUse Add Borders to bring a cropped picture back to an exact frame size for authoring (720x576 for PAL DVD, 720x480 for NTSC) without rescaling it: the picture is centred and the rest filled with bars.", "category": "transform", "icon": "crop", "order": 9, @@ -237,7 +237,7 @@ "default": false, "ui": { "label": "Enable Upscale", - "description": "High-quality upscaling using NNEDI3" + "description": "Enlarge the picture 2x or 4x with one of the upscale methods below, instead of resizing to a target resolution" } }, "upscaleMethod": { @@ -262,7 +262,7 @@ "realCugan": "Real-CUGAN", "animeJanai": "AnimeJaNai (download, non-commercial)" }, - "description": "Edge-directed doubling. NNEDI3 is the usual choice; EEDI3 is slower and follows near-horizontal edges better (it uses NNEDI3 to guide its search); Spline36 is plain resampling with no edge direction. Real-ESRGAN and Real-CUGAN are neural upscalers trained on anime: they redraw clean line art rather than interpolating it, and run on the graphics card (Vulkan). Real-ESRGAN is sharper and faster; Real-CUGAN is gentler and can remove noise as it goes. AnimeJaNai, trained on HD anime, keeps colour exactly and comes in three speeds; it is a separate download, free for non-commercial use only. Use them after deinterlacing: on a combed picture they sharpen the combing too", + "description": "How the picture is enlarged. NNEDI3 is the usual choice, an edge-directed doubler that runs anywhere; EEDI3 is slower and follows near-horizontal edges better (it uses NNEDI3 to guide its search); Spline36 is plain resampling with no edge direction. Real-ESRGAN and Real-CUGAN are neural upscalers trained on anime: they redraw clean line art rather than interpolating it, and run on the graphics card (Vulkan). Real-ESRGAN is sharper and faster; Real-CUGAN is gentler and can remove noise as it goes. AnimeJaNai, trained on HD anime, keeps colour almost exactly and comes in three speeds; it is a separate download, free for non-commercial use only. Use them after deinterlacing: on a combed picture they sharpen the combing too", "widget": "dropdown", "visibleWhen": { "useIntegerUpscale": true From b1e8dd843c762c411fb82b7493cf58454806149f Mon Sep 17 00:00:00 2001 From: Stuart Cameron Date: Sun, 25 Oct 2026 14:19:12 +1100 Subject: [PATCH 06/15] ci: lavapipe on the macOS arm64 runner too; longer GPU probe test timeouts The first CI run with VAPOURBOX_EXPECT_VULKAN=1 showed MoltenVK can't open the hosted arm64 runner's Metal device (it segfaults creating the instance; the runners are VMs). macOS now gets Homebrew's mesa lavapipe through the Khronos loader, which ncnn tries before the preloaded MoltenVK. Only the loader is exposed, on the fallback path, so no other Homebrew library can stand in for a bundled one. On lavapipe the first probe took 30.4 s on Windows, past the default test timeout, so the probe tests now allow 3 minutes and the script test 5 (it may also download AnimeJaNai). Co-Authored-By: Claude Opus 5.5 --- .github/scripts/provision-vulkan.sh | 27 ++++++++++++++++++---- CLAUDE.md | 12 ++++++---- app/test/integration_gpu_plugins_test.dart | 6 +++-- docs/ENGINEERING_NOTES.md | 13 +++++++---- 4 files changed, 44 insertions(+), 14 deletions(-) diff --git a/.github/scripts/provision-vulkan.sh b/.github/scripts/provision-vulkan.sh index 869c8594..4f5c4b7e 100755 --- a/.github/scripts/provision-vulkan.sh +++ b/.github/scripts/provision-vulkan.sh @@ -2,10 +2,11 @@ # Give a CI runner a Vulkan device, so the ncnn-on-Vulkan features (RIFE, and # Real-ESRGAN / Real-CUGAN upscaling) are actually exercised rather than skipped. # -# The hosted runners have no GPU. Linux and Windows get Mesa's lavapipe, a -# Vulkan driver that runs on the CPU — slow, but the same plugin, model and -# script path a user's GPU takes, and ncnn accepts a CPU device. macOS arm64 -# needs nothing: the bundle's MoltenVK runs on the runner's virtual Metal GPU. +# The hosted runners have no GPU, so every one gets Mesa's lavapipe, a Vulkan +# driver that runs on the CPU — slow, but the same plugin, model and script +# path a user's GPU takes, and ncnn accepts a CPU device. That includes macOS +# arm64: those runners are VMs whose Metal device MoltenVK cannot open (it +# segfaults creating the instance — CI run 40155693557, 2026-10-25). # # Wherever a device is provisioned this sets VAPOURBOX_EXPECT_VULKAN=1, and the # GPU tests then FAIL rather than skip if a feature reports itself unavailable @@ -56,6 +57,24 @@ case "$platform" in echo "VAPOURBOX_EXPECT_VULKAN=1" >> "$GITHUB_ENV" ;; macos-arm64) + # Homebrew's mesa builds lavapipe on macOS (vulkan-drivers=swrast), and + # vulkan-loader is the Khronos loader. ncnn dlopens libvulkan.1.dylib before + # falling back to the MoltenVK the scripts preload, so with the loader + # findable both plugins take lavapipe through it — the path any Mac with a + # loader installed takes. Only the loader is exposed, in a directory of its + # own on the *fallback* path: putting Homebrew's lib/ there would let it + # stand in for a library missing from the bundle and hide that. + # Not pinned: Homebrew has no version pins (bottles exist for macos-15). + brew install --quiet mesa vulkan-loader vulkan-tools + prefix="$(brew --prefix)" + icd="$(ls "$prefix"/share/vulkan/icd.d/lvp_icd*.json | head -1)" + dir="$RUNNER_TEMP/vulkan" + rm -rf "$dir" && mkdir -p "$dir" + cp -L "$prefix/lib/libvulkan.1.dylib" "$dir/" + echo "DYLD_FALLBACK_LIBRARY_PATH=$dir:/usr/local/lib:/usr/lib" >> "$GITHUB_ENV" + echo "VK_DRIVER_FILES=$icd" >> "$GITHUB_ENV" + echo "VK_ICD_FILENAMES=$icd" >> "$GITHUB_ENV" + VK_DRIVER_FILES="$icd" vulkaninfo --summary || true echo "VAPOURBOX_EXPECT_VULKAN=1" >> "$GITHUB_ENV" ;; *) diff --git a/CLAUDE.md b/CLAUDE.md index 5ec014b7..699fef32 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1126,10 +1126,14 @@ than fail cleanly. place they are published), extracted in one pass and each checked by hash. Every pin and hash is in all three `download-deps-*` scripts. - **Tested for real in CI.** The hosted runners have no GPU, so - `.github/scripts/provision-vulkan.sh` gives Linux and Windows Mesa's - lavapipe (a CPU Vulkan driver ncnn accepts) and sets - `VAPOURBOX_EXPECT_VULKAN=1` wherever a device exists (macOS arm64 uses the - runner's Metal GPU through MoltenVK). Under that variable every GPU test + `.github/scripts/provision-vulkan.sh` gives every platform that ships the + features Mesa's lavapipe (a CPU Vulkan driver ncnn accepts) and sets + `VAPOURBOX_EXPECT_VULKAN=1`. That includes macOS arm64: the hosted runners + are VMs whose Metal device MoltenVK cannot open (it segfaults), so they get + Homebrew's mesa + the Khronos loader, which ncnn tries before the preloaded + MoltenVK; only the loader goes on `DYLD_FALLBACK_LIBRARY_PATH`, so nothing + else of Homebrew's can stand in for a bundled library. MoltenVK itself is + therefore only exercised on real Macs. Under that variable every GPU test **fails** rather than skips if a feature reports itself unavailable (`app/test/support/gpu_expectations.dart`), so a broken bundle cannot hide behind "no GPU". `integration_gpu_plugins_test.dart` (push gate) probes, diff --git a/app/test/integration_gpu_plugins_test.dart b/app/test/integration_gpu_plugins_test.dart index 62ac7155..349c5743 100644 --- a/app/test/integration_gpu_plugins_test.dart +++ b/app/test/integration_gpu_plugins_test.dart @@ -149,7 +149,9 @@ void main() { reason: 'the probe could not even run here, which hides the real ' 'reason from the user: ${status.message}'); } - }); + // The first probe initialises the device and compiles shaders; on a + // CPU Vulkan driver (lavapipe on CI) that took 30.4 s on Windows. + }, timeout: const Timeout(Duration(minutes: 3))); }); } @@ -205,7 +207,7 @@ void main() { if (platformId.startsWith('macos')) { expect(script, contains('moltenvk'), reason: 'MoltenVK must be preloaded on macOS'); } - }); + }, timeout: const Timeout(Duration(minutes: 5))); test('where it runs, a real frame comes out of the GPU at the right size', () async { if (!await requireGpuPlugin(f.plugin.probeFlag)) return; diff --git a/docs/ENGINEERING_NOTES.md b/docs/ENGINEERING_NOTES.md index 072f032d..5398b7bd 100644 --- a/docs/ENGINEERING_NOTES.md +++ b/docs/ENGINEERING_NOTES.md @@ -2474,10 +2474,15 @@ Pixel checks (heavy test), against a plain Spline36 upscale of the same frames: centre on both axes (0.75 px), and that Spline36 itself lands on factor x source (within 0.25 px), so the reference is known-good. -CI runs it all for real: Mesa lavapipe on the GPU-less Linux/Windows runners -(ncnn tags a CPU device as type 3 but does not skip it), the runner's Metal GPU -on macOS arm64, and `VAPOURBOX_EXPECT_VULKAN=1` so an unavailable feature is a -failure there, not a skip. +CI runs it all for real: Mesa lavapipe on every GPU-less runner (ncnn tags a +CPU device as type 3 but does not skip it) and `VAPOURBOX_EXPECT_VULKAN=1`, so +an unavailable feature is a failure there, not a skip. The first plan used the +macOS arm64 runner's Metal GPU through MoltenVK; on the first run with the +variable set, both probes died with SIGSEGV creating the instance — those +runners are VMs — so macOS gets Homebrew's lavapipe too, reached through the +Khronos loader that ncnn dlopens before falling back to MoltenVK. MoltenVK is +therefore tested only on real Macs. The first probe on lavapipe takes ~30 s +(Windows: 30.4 s), past flutter_test's default timeout. ### Descale and native-resolution detection (2026-10-25) From 36fe58948a4b831bfb76fd605bb83e9409df824c Mon Sep 17 00:00:00 2001 From: Stuart Cameron Date: Sun, 25 Oct 2026 14:43:01 +1100 Subject: [PATCH 07/15] ci: macOS loader via ~/lib; crop the push-gate upscale frames - SIP strips DYLD_* at the bundle's /bin/bash vspipe wrapper, so the loader is now copied into ~/lib, which is on dyld's default fallback path. - On Windows lavapipe one 720x576 Real-CUGAN frame took over 5 minutes, so the push-gate upscale jobs crop to 240x192 first. Full-size checks stay in the heavy suite. Co-Authored-By: Claude Opus 5.5 --- .github/scripts/provision-vulkan.sh | 15 ++++++++------- CLAUDE.md | 6 ++++-- app/test/integration_gpu_plugins_test.dart | 15 ++++++++++++--- 3 files changed, 24 insertions(+), 12 deletions(-) diff --git a/.github/scripts/provision-vulkan.sh b/.github/scripts/provision-vulkan.sh index 4f5c4b7e..ab3a45ca 100755 --- a/.github/scripts/provision-vulkan.sh +++ b/.github/scripts/provision-vulkan.sh @@ -61,17 +61,18 @@ case "$platform" in # vulkan-loader is the Khronos loader. ncnn dlopens libvulkan.1.dylib before # falling back to the MoltenVK the scripts preload, so with the loader # findable both plugins take lavapipe through it — the path any Mac with a - # loader installed takes. Only the loader is exposed, in a directory of its - # own on the *fallback* path: putting Homebrew's lib/ there would let it - # stand in for a library missing from the bundle and hide that. + # loader installed takes. Only the loader is exposed, in ~/lib — the first + # directory on dyld's *default* fallback path — never Homebrew's lib/, + # which could stand in for a library missing from the bundle and hide + # that. Not DYLD_FALLBACK_LIBRARY_PATH: the bundle's vspipe is a /bin/bash + # wrapper, and SIP strips every DYLD_* variable when a protected binary + # runs, so it never reached the plugins (CI run 40157771950). # Not pinned: Homebrew has no version pins (bottles exist for macos-15). brew install --quiet mesa vulkan-loader vulkan-tools prefix="$(brew --prefix)" icd="$(ls "$prefix"/share/vulkan/icd.d/lvp_icd*.json | head -1)" - dir="$RUNNER_TEMP/vulkan" - rm -rf "$dir" && mkdir -p "$dir" - cp -L "$prefix/lib/libvulkan.1.dylib" "$dir/" - echo "DYLD_FALLBACK_LIBRARY_PATH=$dir:/usr/local/lib:/usr/lib" >> "$GITHUB_ENV" + mkdir -p "$HOME/lib" + cp -L "$prefix/lib/libvulkan.1.dylib" "$HOME/lib/" echo "VK_DRIVER_FILES=$icd" >> "$GITHUB_ENV" echo "VK_ICD_FILENAMES=$icd" >> "$GITHUB_ENV" VK_DRIVER_FILES="$icd" vulkaninfo --summary || true diff --git a/CLAUDE.md b/CLAUDE.md index 699fef32..b424e805 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1131,8 +1131,10 @@ than fail cleanly. `VAPOURBOX_EXPECT_VULKAN=1`. That includes macOS arm64: the hosted runners are VMs whose Metal device MoltenVK cannot open (it segfaults), so they get Homebrew's mesa + the Khronos loader, which ncnn tries before the preloaded - MoltenVK; only the loader goes on `DYLD_FALLBACK_LIBRARY_PATH`, so nothing - else of Homebrew's can stand in for a bundled library. MoltenVK itself is + MoltenVK. Only the loader is exposed, copied into `~/lib` (dyld's default + fallback path), so nothing else of Homebrew's can stand in for a bundled + library — not via `DYLD_*` variables, which SIP strips at the bundle's + `/bin/bash` vspipe wrapper. MoltenVK itself is therefore only exercised on real Macs. Under that variable every GPU test **fails** rather than skips if a feature reports itself unavailable (`app/test/support/gpu_expectations.dart`), so a broken bundle cannot hide diff --git a/app/test/integration_gpu_plugins_test.dart b/app/test/integration_gpu_plugins_test.dart index 349c5743..8f52ab67 100644 --- a/app/test/integration_gpu_plugins_test.dart +++ b/app/test/integration_gpu_plugins_test.dart @@ -70,8 +70,17 @@ VideoJob _upscaleJob(UpscaleMethod method, {String noise = 'conservative'}) => _ 'upscale_${method.name}', ProcessingPipeline( deinterlace: const QTGMCParameters(enabled: false), + // Cropped to 240x192 first: on a CPU Vulkan driver (lavapipe, on the + // GPU-less CI runners) one whole 720x576 Real-CUGAN frame took over + // five minutes on Windows. This test only needs a real frame out of + // the GPU at the right size; full-size encodes are the heavy suite's. cropResize: CropResizeParameters( enabled: true, + cropEnabled: true, + cropLeft: 240, + cropRight: 240, + cropTop: 192, + cropBottom: 192, useIntegerUpscale: true, upscaleMethod: method, upscaleFactor: 2, @@ -106,17 +115,17 @@ final _features = { 'Real-ESRGAN': _Feature(GpuPlugin.upscale, () => _upscaleJob(UpscaleMethod.realEsrgan), 'ncnn', [ 'RealESRGANv2-animevideo-xsx2.onnx', 'core.ncnn.Model(_ai_rgb, network_path=_ai_model, fp16=True', - ], (1440, 1152)), + ], (480, 384)), 'Real-CUGAN': _Feature(GpuPlugin.upscale, () => _upscaleJob(UpscaleMethod.realCugan, noise: 'denoise3x'), 'ncnn', [ 'up2x-latest-denoise3x.onnx', 'core.ncnn.Model(_ai_rgb, network_path=_ai_model, fp16=True', - ], (1440, 1152)), + ], (480, 384)), 'AnimeJaNai': _Feature(GpuPlugin.upscale, () => _upscaleJob(UpscaleMethod.animeJanai), 'animejanai', [ '2x_AnimeJaNai_SD_V1beta34_Compact.onnx', '2x_AnimeJaNai_HD_V3_Compact.onnx', '_ai_model = _ai_model[0] if _ai_rgb.height <= 576 else _ai_model[1]', 'core.ncnn.Model(_ai_rgb, network_path=_ai_model, fp16=True', - ], (1440, 1152), needsAnimeJanai: true), + ], (480, 384), needsAnimeJanai: true), }; void main() { From f898a846560f554ada6bb0fb7eee0d08f604612b Mon Sep 17 00:00:00 2001 From: Stuart Cameron Date: Sun, 25 Oct 2026 14:43:02 +1100 Subject: [PATCH 08/15] =?UTF-8?q?feat(ui):=20NEW=20badges=20lead=20to=20wh?= =?UTF-8?q?at's=20new=20=E2=80=94=20options,=20methods,=20sections,=20swit?= =?UTF-8?q?ches?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - A new enum option (optionSinceAppVersion) is badged in its dropdown, and a new method (sinceAppVersion on the method) in the method list. - Every control on the way to something new is badged too: a parameter whose option is new or that reveals a new control (through every visibleWhen level, so "Enable Upscale" badges), the Method label, a section heading, and the pass row. The panels pass the schema to the labels through ParameterBadgeScope. - The new Settings → Add-ons tab is badged, and the Settings button gets a dot while it is new. - Restamped: Real-ESRGAN, Real-CUGAN and AnimeJaNai are now stamped as options rather than on upscaleMethod; RIFE, BM3D and KNLMeans as methods rather than on the method parameter. Noise Reduction's stamp had been on a hidden parameter, so it badged nothing. - Tests: the derived rules; lints for option and method stamps, and against stamping a hidden parameter; the real 1.5.0 badges on the shipped schemas; and a widget test of the Crop & Resize panel. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 9 +- app/assets/filters/core/crop_resize.json | 6 +- app/assets/filters/core/frame_rate.json | 2 +- app/assets/filters/core/noise_reduction.json | 5 +- app/lib/models/filter_schema.dart | 13 +++ app/lib/services/whats_new_service.dart | 48 +++++++- app/lib/views/main_window.dart | 12 +- .../views/settings/addons_settings_tab.dart | 3 + .../views/settings/dynamic_filter_panel.dart | 62 ++++++++--- app/lib/views/settings/settings_dialog.dart | 24 +++- .../settings/widgets/parameter_widgets.dart | 62 +++++++++-- app/test/filter_schema_curation_test.dart | 46 ++++++++ app/test/new_badges_shipped_test.dart | 76 +++++++++++++ app/test/new_badges_widget_test.dart | 86 +++++++++++++++ app/test/whats_new_service_test.dart | 104 ++++++++++++++++++ docs/FILTER_SCHEMA.md | 30 ++++- 16 files changed, 540 insertions(+), 48 deletions(-) create mode 100644 app/test/new_badges_shipped_test.dart create mode 100644 app/test/new_badges_widget_test.dart diff --git a/CLAUDE.md b/CLAUDE.md index b424e805..e3f2ee65 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -588,7 +588,8 @@ or `UiSection` directly (both silently drop it if written one level up). `visibleWhen` outside `ui`, and every condition must name a parameter/method the filter actually has. -**`sinceAppVersion`** (filter-level or per-parameter, e.g. `"1.2.0"`): tags +**`sinceAppVersion`** (filter-level, per-parameter, per-method, or per enum +option via `optionSinceAppVersion`, e.g. `"1.2.0"`): tags when a filter or parameter shipped, purely so `WhatsNewService` can draw a "NEW" badge next to it — persisting across every launch of the current app version, and only advancing on the next real update, not clearing after one @@ -597,7 +598,11 @@ the version that change will actually ship in** (a minor bump on the latest public release, or whatever's already in `app/pubspec.yaml` if that's ahead) — not retroactively at release time, which is exactly how properties get missed. Never touch the stamp on anything already shipped — see "NEW badges" -in [docs/FILTER_SCHEMA.md](docs/FILTER_SCHEMA.md). +in [docs/FILTER_SCHEMA.md](docs/FILTER_SCHEMA.md). **Stamp only what is new** +(the option, not the parameter it was added to; never a hidden parameter): +every control leading to it badges automatically — its dropdown, the switch +that reveals it, its section, the pass row, and for a new Settings tab the +Settings button. Several ongoing rules for schemas and the panel that renders them, each guarded by `filter_schema_curation_test.dart` unless noted — see diff --git a/app/assets/filters/core/crop_resize.json b/app/assets/filters/core/crop_resize.json index 342c43df..3d3ef4a1 100644 --- a/app/assets/filters/core/crop_resize.json +++ b/app/assets/filters/core/crop_resize.json @@ -268,7 +268,11 @@ "useIntegerUpscale": true } }, - "sinceAppVersion": "1.5.0" + "optionSinceAppVersion": { + "realEsrgan": "1.5.0", + "realCugan": "1.5.0", + "animeJanai": "1.5.0" + } }, "upscaleFactor": { "type": "integer", diff --git a/app/assets/filters/core/frame_rate.json b/app/assets/filters/core/frame_rate.json index 0add7701..add60912 100644 --- a/app/assets/filters/core/frame_rate.json +++ b/app/assets/filters/core/frame_rate.json @@ -19,6 +19,7 @@ }, { "id": "rife", + "sinceAppVersion": "1.5.0", "name": "AI interpolation (RIFE)", "description": "Builds the new frames with the RIFE 4.26 neural network on the GPU. Somewhat more accurate than motion interpolation, several times slower; needs Vulkan.", "function": "core.rife.RIFE", @@ -48,7 +49,6 @@ "rife", "duplicate" ], - "sinceAppVersion": "1.5.0", "ui": { "label": "Method", "description": "How the new frames are produced.", diff --git a/app/assets/filters/core/noise_reduction.json b/app/assets/filters/core/noise_reduction.json index 9070ab78..2e7e0585 100644 --- a/app/assets/filters/core/noise_reduction.json +++ b/app/assets/filters/core/noise_reduction.json @@ -177,6 +177,7 @@ }, { "id": "bm3d", + "sinceAppVersion": "1.5.0", "name": "BM3D", "description": "Block-matching 3D denoiser: finds similar patches across the frame (and neighbouring frames) and cleans them together. Leaves flat colour areas very clean without smearing line art, which is why it is the usual choice for anime. CPU, moderate speed", "function": "core.bm3d.VBasic + core.bm3d.VFinal", @@ -191,6 +192,7 @@ }, { "id": "knlmeans", + "sinceAppVersion": "1.5.0", "name": "KNLMeansCL (GPU)", "description": "Non-local means on the graphics card. Similar results to BM3D and much faster, but it needs a GPU with working OpenCL; where there is none, VapourBox says so and the job is refused", "function": "core.knlm.KNLMeansCL", @@ -234,8 +236,7 @@ ], "ui": { "hidden": true - }, - "sinceAppVersion": "1.5.0" + } }, "smDegrainTr": { "type": "integer", diff --git a/app/lib/models/filter_schema.dart b/app/lib/models/filter_schema.dart index 7089ddee..b600049a 100644 --- a/app/lib/models/filter_schema.dart +++ b/app/lib/models/filter_schema.dart @@ -139,6 +139,13 @@ class ParameterDefinition { /// isn't worth flagging (most edits aren't). final String? sinceAppVersion; + /// The same, per enum option: `{"realEsrgan": "1.5.0"}`. For options added + /// to a parameter that already shipped — the parameter itself is not new, so + /// it carries no [sinceAppVersion]; the option is badged in the list and the + /// parameter's label badges because it holds one (see + /// [WhatsNewService.isParameterNew]). + final Map? optionSinceAppVersion; + const ParameterDefinition({ required this.type, required this.defaultValue, @@ -150,6 +157,7 @@ class ParameterDefinition { this.ui, this.optional, this.sinceAppVersion, + this.optionSinceAppVersion, }); /// Get the VapourSynth parameter name (falls back to schema name if not specified). @@ -212,6 +220,10 @@ class MethodDefinition { /// [FilterSchema.visibleMethods]. final bool advancedOnly; + /// The app version this method was added in, for its "NEW" badge in the + /// method list (and the Method label's, and the pass row's). + final String? sinceAppVersion; + const MethodDefinition({ required this.id, required this.name, @@ -219,6 +231,7 @@ class MethodDefinition { required this.function, required this.parameters, this.advancedOnly = false, + this.sinceAppVersion, }); factory MethodDefinition.fromJson(Map json) => diff --git a/app/lib/services/whats_new_service.dart b/app/lib/services/whats_new_service.dart index 6fde80cb..9ce41939 100644 --- a/app/lib/services/whats_new_service.dart +++ b/app/lib/services/whats_new_service.dart @@ -92,10 +92,54 @@ class WhatsNewService { bool isSchemaNew(FilterSchema? schema) { if (schema == null) return false; if (isNew(schema.sinceAppVersion)) return true; - return schema.parameters.values.any( - (p) => p.ui?.hidden != true && isNew(p.sinceAppVersion)); + if (isMethodChoiceNew(schema)) return true; + return schema.parameters.entries + .any((e) => e.value.ui?.hidden != true && isParameterNew(schema, e.key)); } + // The rule everywhere: something new is badged, and so is every control on + // the way to it — so a badge is never only inside a closed dropdown, a + // collapsed section or behind a switch that is off. + + /// Whether one of [param]'s enum options is new. + bool isOptionNew(ParameterDefinition param, String option) => + isNew(param.optionSinceAppVersion?[option]); + + /// Whether [method] is new. + bool isMethodNew(MethodDefinition method) => isNew(method.sinceAppVersion); + + /// Whether parameter [id]'s control should carry a badge: it is new itself, + /// one of its options is, or a control it reveals is — any parameter whose + /// `visibleWhen` names it ("Enable Upscale" badges for the new upscalers' + /// settings it shows). + bool isParameterNew(FilterSchema schema, String id, [Set? visiting]) { + final param = schema.parameters[id]; + if (param == null) return false; + if (isNew(param.sinceAppVersion)) return true; + if (param.optionSinceAppVersion?.values.any(isNew) ?? false) return true; + final seen = visiting ?? {}; + if (!seen.add(id)) return false; + return _dependents(schema, id).any((d) => isParameterNew(schema, d, seen)); + } + + /// Whether the method dropdown should carry a badge: a method is new, or a + /// control that only some methods show is. + bool isMethodChoiceNew(FilterSchema schema) => + schema.methods.any(isMethodNew) || + _dependents(schema, 'method').any((d) => isParameterNew(schema, d, {'method'})); + + /// Whether a section's title should carry a badge: anything it holds does. + bool isSectionNew(FilterSchema schema, UiSection section) => section.parameters + .any((id) => id == 'method' ? isMethodChoiceNew(schema) : isParameterNew(schema, id)); + + /// The visible parameters whose `visibleWhen` depends on [id]. + Iterable _dependents(FilterSchema schema, String id) => schema.parameters.entries + .where((e) => + e.key != id && + e.value.ui?.hidden != true && + (e.value.ui?.visibleWhen?.containsKey(id) ?? false)) + .map((e) => e.key); + /// Reset to the unloaded default. Tests only — the singleton outlives a /// single test case otherwise. @visibleForTesting diff --git a/app/lib/views/main_window.dart b/app/lib/views/main_window.dart index 87224217..2b6bdf6b 100644 --- a/app/lib/views/main_window.dart +++ b/app/lib/views/main_window.dart @@ -30,6 +30,8 @@ import 'queue_panel.dart'; import 'settings/settings_dialog.dart'; import '../widgets/resizable_split.dart'; import '../widgets/warning_banner.dart'; +import '../services/whats_new_service.dart'; +import 'settings/addons_settings_tab.dart'; class MainWindow extends StatelessWidget { const MainWindow({super.key}); @@ -408,7 +410,15 @@ class MainWindow extends StatelessWidget { // Settings button — About and bug reporting live in its General tab IconButton( - icon: const Icon(Icons.settings), + // Marked in the NEW badge's colour while a Settings tab is new, so + // the badge inside is not the only way to find it. + icon: Badge( + isLabelVisible: + WhatsNewService.instance.isNew(AddonsSettingsTab.sinceAppVersion), + smallSize: 8, + backgroundColor: Theme.of(context).colorScheme.tertiary, + child: const Icon(Icons.settings), + ), tooltip: AppShortcut.openSettings.tooltip, onPressed: () => _showSettings(context, viewModel), ), diff --git a/app/lib/views/settings/addons_settings_tab.dart b/app/lib/views/settings/addons_settings_tab.dart index 5b156d21..4d86f4c4 100644 --- a/app/lib/views/settings/addons_settings_tab.dart +++ b/app/lib/views/settings/addons_settings_tab.dart @@ -11,6 +11,9 @@ class AddonsSettingsTab extends StatefulWidget { final AddonService? service; const AddonsSettingsTab({super.key, this.service}); + /// The app version this tab shipped in, for its "NEW" badge. + static const sinceAppVersion = '1.5.0'; + @override State createState() => _AddonsSettingsTabState(); } diff --git a/app/lib/views/settings/dynamic_filter_panel.dart b/app/lib/views/settings/dynamic_filter_panel.dart index e0070c0d..fe8b75d2 100644 --- a/app/lib/views/settings/dynamic_filter_panel.dart +++ b/app/lib/views/settings/dynamic_filter_panel.dart @@ -4,6 +4,8 @@ import 'package:provider/provider.dart'; import '../../models/dynamic_parameters.dart'; import '../../models/filter_schema.dart'; import '../../services/advanced_mode_service.dart'; +import '../../services/whats_new_service.dart'; +import '../../widgets/new_badge.dart'; import 'widgets/parameter_widgets.dart'; /// A dynamically-generated settings panel based on a filter schema. @@ -33,7 +35,10 @@ class DynamicFilterPanel extends StatelessWidget { }); @override - Widget build(BuildContext context) { + Widget build(BuildContext context) => + ParameterBadgeScope(schema: schema, child: _buildBody(context)); + + Widget _buildBody(BuildContext context) { return Column( crossAxisAlignment: CrossAxisAlignment.start, children: [ @@ -92,9 +97,10 @@ class DynamicFilterPanel extends StatelessWidget { return Card( margin: EdgeInsets.zero, child: ExpansionTile( - title: Text( - section.title, - style: Theme.of(context).textTheme.titleSmall, + title: _sectionTitle( + schema, + section, + Text(section.title, style: Theme.of(context).textTheme.titleSmall), ), initiallyExpanded: section.expanded, childrenPadding: const EdgeInsets.all(16), @@ -204,7 +210,10 @@ class DynamicFilterPanelCompact extends StatelessWidget { }); @override - Widget build(BuildContext context) { + Widget build(BuildContext context) => + ParameterBadgeScope(schema: schema, child: _buildBody(context)); + + Widget _buildBody(BuildContext context) { final advancedMode = context.watch().enabled; // The method the pipeline is actually using, which is always offered even @@ -253,7 +262,10 @@ class DynamicFilterPanelCompact extends StatelessWidget { // Method selection (if multiple methods available) if (hasMultipleMethods) ...[ - Text('Method', style: Theme.of(context).textTheme.labelLarge), + _withBadge( + Text('Method', style: Theme.of(context).textTheme.labelLarge), + WhatsNewService.instance.isMethodChoiceNew(schema), + ), const SizedBox(height: 8), DropdownButtonFormField( value: selectedMethod, @@ -265,8 +277,9 @@ class DynamicFilterPanelCompact extends StatelessWidget { items: visibleMethods.map((method) { return DropdownMenuItem( value: method.id, - child: Text( + child: optionLabelWithBadge( method.advancedOnly ? '${method.name} (advanced)' : method.name, + WhatsNewService.instance.isMethodNew(method), ), ); }).toList(), @@ -404,15 +417,19 @@ class DynamicFilterPanelCompact extends StatelessWidget { top: widgets.isEmpty ? 0 : 16, bottom: 8, ), - child: Text( - section.title, - style: Theme.of(context).textTheme.titleSmall?.copyWith( - color: section.advancedOnly - ? Theme.of(context).colorScheme.primary - : Theme.of(context) - .colorScheme - .onSurface - .withValues(alpha: 0.7), + child: _sectionTitle( + schema, + section, + Text( + section.title, + style: Theme.of(context).textTheme.titleSmall?.copyWith( + color: section.advancedOnly + ? Theme.of(context).colorScheme.primary + : Theme.of(context) + .colorScheme + .onSurface + .withValues(alpha: 0.7), + ), ), ), ), @@ -611,3 +628,16 @@ class DynamicFilterPanelCompact extends StatelessWidget { return true; } } + +/// A section heading, badged when anything in the section is new — a +/// collapsed section would otherwise hide every badge inside it. +Widget _sectionTitle(FilterSchema schema, UiSection section, Widget title) => + _withBadge(title, WhatsNewService.instance.isSectionNew(schema, section)); + +Widget _withBadge(Widget child, bool isNew) { + if (!isNew) return child; + return Row( + mainAxisSize: MainAxisSize.min, + children: [Flexible(child: child), const SizedBox(width: 6), const NewBadge()], + ); +} diff --git a/app/lib/views/settings/settings_dialog.dart b/app/lib/views/settings/settings_dialog.dart index 753b13a1..ad228dac 100644 --- a/app/lib/views/settings/settings_dialog.dart +++ b/app/lib/views/settings/settings_dialog.dart @@ -364,12 +364,24 @@ class _SettingsDialogState extends State // Tab bar TabBar( controller: _tabController, - tabs: const [ - Tab(text: 'General'), - Tab(text: 'Output'), - Tab(text: 'Input'), - Tab(text: 'Add-ons'), - Tab(text: 'Changes'), + tabs: [ + const Tab(text: 'General'), + const Tab(text: 'Output'), + const Tab(text: 'Input'), + Tab( + child: Row( + mainAxisSize: MainAxisSize.min, + children: [ + const Text('Add-ons'), + if (WhatsNewService.instance + .isNew(AddonsSettingsTab.sinceAppVersion)) ...[ + const SizedBox(width: 6), + const NewBadge(), + ], + ], + ), + ), + const Tab(text: 'Changes'), ], ), diff --git a/app/lib/views/settings/widgets/parameter_widgets.dart b/app/lib/views/settings/widgets/parameter_widgets.dart index 8e3e710a..e6685fe0 100644 --- a/app/lib/views/settings/widgets/parameter_widgets.dart +++ b/app/lib/views/settings/widgets/parameter_widgets.dart @@ -7,18 +7,40 @@ import '../../../services/whats_new_service.dart'; import '../../../viewmodels/main_viewmodel.dart'; import '../../../widgets/new_badge.dart'; -/// A parameter's label, with a "NEW" badge appended when -/// [WhatsNewService.isNew] flags [param.sinceAppVersion]. Every widget below -/// renders its label through this instead of a bare `Text` so the badge shows -/// up consistently regardless of widget type. +/// Gives the parameter widgets below the schema they belong to, so a label can +/// badge for what its control leads to (a new option, a control it reveals), +/// not just for itself — see [WhatsNewService.isParameterNew]. The filter +/// panels wrap their controls in one. +class ParameterBadgeScope extends InheritedWidget { + final FilterSchema schema; + const ParameterBadgeScope({super.key, required this.schema, required super.child}); + + static FilterSchema? maybeOf(BuildContext context) => + context.dependOnInheritedWidgetOfExactType()?.schema; + + @override + bool updateShouldNotify(ParameterBadgeScope old) => old.schema != schema; +} + +/// A parameter's label, with a "NEW" badge appended when the parameter, one of +/// its options, or a control it reveals is new. Every widget below renders its +/// label through this instead of a bare `Text` so the badge shows up +/// consistently regardless of widget type. Widget _paramLabel( BuildContext context, String label, + String paramId, ParameterDefinition param, { TextStyle? style, }) { final text = Text(label, style: style); - if (!WhatsNewService.instance.isNew(param.sinceAppVersion)) return text; + final whatsNew = WhatsNewService.instance; + final schema = ParameterBadgeScope.maybeOf(context); + final isNew = schema != null && schema.parameters.containsKey(paramId) + ? whatsNew.isParameterNew(schema, paramId) + : whatsNew.isNew(param.sinceAppVersion) || + (param.optionSinceAppVersion?.values.any(whatsNew.isNew) ?? false); + if (!isNew) return text; return Row( mainAxisSize: MainAxisSize.min, @@ -30,6 +52,19 @@ Widget _paramLabel( ); } +/// An option or method in a dropdown, badged when it is new. +Widget optionLabelWithBadge(String text, bool isNew) { + if (!isNew) return Text(text, overflow: TextOverflow.ellipsis); + return Row( + mainAxisSize: MainAxisSize.min, + children: [ + Flexible(child: Text(text, overflow: TextOverflow.ellipsis)), + const SizedBox(width: 6), + const NewBadge(), + ], + ); +} + /// Factory for creating parameter widgets based on schema definition. class ParameterWidgetFactory { /// Build a widget for the given parameter definition. @@ -264,6 +299,7 @@ class _SliderParameterWidget extends StatelessWidget { _paramLabel( context, '$label: ${doubleValue.toStringAsFixed(precision)}', + paramId, param, style: Theme.of(context).textTheme.labelLarge, ), @@ -320,7 +356,7 @@ class _DropdownParameterWidget extends StatelessWidget { return Column( crossAxisAlignment: CrossAxisAlignment.start, children: [ - _paramLabel(context, label, param, + _paramLabel(context, label, paramId, param, style: Theme.of(context).textTheme.labelLarge), const SizedBox(height: 8), DropdownButtonFormField( @@ -333,7 +369,9 @@ class _DropdownParameterWidget extends StatelessWidget { items: options.map((option) { return DropdownMenuItem( value: option, - child: Text(param.ui?.optionLabels?[option] ?? _formatOptionName(option)), + child: optionLabelWithBadge( + param.ui?.optionLabels?[option] ?? _formatOptionName(option), + WhatsNewService.instance.isOptionNew(param, option)), ); }).toList(), onChanged: (newValue) { @@ -384,7 +422,7 @@ class _CheckboxParameterWidget extends StatelessWidget { return Column( crossAxisAlignment: CrossAxisAlignment.start, children: [ - _paramLabel(context, label, param, + _paramLabel(context, label, paramId, param, style: Theme.of(context).textTheme.labelLarge), const SizedBox(height: 8), DropdownButtonFormField( @@ -419,7 +457,7 @@ class _CheckboxParameterWidget extends StatelessWidget { // Default: render as switch return SwitchListTile( - title: _paramLabel(context, label, param), + title: _paramLabel(context, label, paramId, param), subtitle: param.ui?.description != null ? Text(param.ui!.description!) : null, value: boolValue, contentPadding: EdgeInsets.zero, @@ -450,7 +488,7 @@ class _TextFieldParameterWidget extends StatelessWidget { return Column( crossAxisAlignment: CrossAxisAlignment.start, children: [ - _paramLabel(context, label, param, + _paramLabel(context, label, paramId, param, style: Theme.of(context).textTheme.labelLarge), const SizedBox(height: 8), TextFormField( @@ -545,7 +583,7 @@ class _FilePickerParameterWidgetState return Column( crossAxisAlignment: CrossAxisAlignment.start, children: [ - _paramLabel(context, label, widget.param, + _paramLabel(context, label, widget.paramId, widget.param, style: Theme.of(context).textTheme.labelLarge), const SizedBox(height: 8), Row( @@ -599,7 +637,7 @@ class _NumberParameterWidget extends StatelessWidget { return Column( crossAxisAlignment: CrossAxisAlignment.start, children: [ - _paramLabel(context, label, param, + _paramLabel(context, label, paramId, param, style: Theme.of(context).textTheme.labelLarge), const SizedBox(height: 8), TextFormField( diff --git a/app/test/filter_schema_curation_test.dart b/app/test/filter_schema_curation_test.dart index 0c30c717..924e5855 100644 --- a/app/test/filter_schema_curation_test.dart +++ b/app/test/filter_schema_curation_test.dart @@ -97,6 +97,52 @@ void main() { }); }); + // Option and method stamps feed the same compare. And a stamp on a hidden + // parameter can never show: there is no control to badge, so whatever it + // meant to flag goes unannounced (Noise Reduction's new BM3D and KNLMeans + // methods were stamped on the hidden `method` parameter and badged nowhere). + group('option and method stamps', () { + final versionPattern = RegExp(r'^\d+\.\d+\.\d+$'); + rawSchemas.forEach((filename, raw) { + final id = raw['id'] as String; + final parameters = (raw['parameters'] as Map).cast(); + + test('$id: option stamps name real options and look like "1.2.0"', () { + for (final entry in parameters.entries) { + final p = entry.value as Map; + final stamps = (p['optionSinceAppVersion'] as Map?)?.cast(); + if (stamps == null) continue; + final options = (p['options'] as List?)?.cast() ?? const []; + for (final MapEntry(key: option, value: since) in stamps.entries) { + expect(options, contains(option), reason: '${entry.key}: stamp for unknown option $option'); + expect(versionPattern.hasMatch(since as String), isTrue, + reason: '${entry.key}.$option: "$since" is not X.Y.Z'); + } + } + }); + + test('$id: method stamps look like "1.2.0"', () { + for (final m in (raw['methods'] as List? ?? const []).cast()) { + final since = m['sinceAppVersion']; + if (since == null) continue; + expect(versionPattern.hasMatch(since as String), isTrue, + reason: '${m['id']}: "$since" is not X.Y.Z'); + } + }); + + test('$id: no hidden parameter carries a stamp nobody can see', () { + for (final entry in parameters.entries) { + final p = entry.value as Map; + if ((p['ui'] as Map?)?['hidden'] != true) continue; + expect(p['sinceAppVersion'], isNull, + reason: '${entry.key} is hidden, so its badge never shows; stamp ' + 'what is new instead (a method, an option)'); + expect(p['optionSinceAppVersion'], isNull, reason: entry.key); + } + }); + }); + }); + group('method curation', () { for (final schema in schemas) { test('${schema.id}: first method is never advanced-only', () { diff --git a/app/test/new_badges_shipped_test.dart b/app/test/new_badges_shipped_test.dart new file mode 100644 index 00000000..e2d0c2bb --- /dev/null +++ b/app/test/new_badges_shipped_test.dart @@ -0,0 +1,76 @@ +// The 1.5.0 NEW badges on the shipped schemas, as a user upgrading from 1.4.0 +// sees them: every new filter, method, option and setting is badged, and so is +// every control on the way to it. A wrongly placed stamp (on the parameter +// rather than its new option, or on a hidden parameter) passes the format +// lints but badges the wrong thing or nothing, so the result is checked here. +import 'dart:convert'; +import 'dart:io'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:package_info_plus/package_info_plus.dart'; +import 'package:shared_preferences/shared_preferences.dart'; +import 'package:vapourbox/models/filter_schema.dart'; +import 'package:vapourbox/services/whats_new_service.dart'; + +FilterSchema _load(String file) => FilterSchema.fromJson( + jsonDecode(File('assets/filters/core/$file').readAsStringSync()) as Map); + +void main() { + TestWidgetsFlutterBinding.ensureInitialized(); + final service = WhatsNewService.instance; + + setUp(() async { + // Upgraded from 1.4.0 to 1.5.0. + SharedPreferences.setMockInitialValues({ + 'lastRunAppVersion': '1.4.0', + 'lastSeenAppVersion': '1.4.0', + }); + PackageInfo.setMockInitialValues( + appName: 'VapourBox', packageName: 'app.vapourbox.vapourbox', + version: '1.5.0', buildNumber: '1', buildSignature: ''); + service.resetForTesting(); + await service.initialize(); + }); + + test('Crop & Resize: the AI upscalers, their method list and the switch that shows it', () { + final s = _load('crop_resize.json'); + final method = s.parameters['upscaleMethod']!; + for (final o in ['realEsrgan', 'realCugan', 'animeJanai']) { + expect(service.isOptionNew(method, o), isTrue, reason: o); + } + for (final o in ['nnedi3Rpow2', 'eedi3Rpow2', 'spline36']) { + expect(service.isOptionNew(method, o), isFalse, reason: '$o shipped in 1.4.0'); + } + expect(service.isParameterNew(s, 'upscaleMethod'), isTrue); + expect(service.isParameterNew(s, 'useIntegerUpscale'), isTrue, + reason: 'Enable Upscale reveals the new upscalers'); + for (final p in ['cuganNoise', 'animejanaiModel', 'animejanaiSharp']) { + expect(service.isParameterNew(s, p), isTrue, reason: p); + } + final section = s.ui!.sections!.firstWhere((x) => x.parameters.contains('upscaleMethod')); + expect(service.isSectionNew(s, section), isTrue); + expect(service.isSchemaNew(s), isTrue); + expect(service.isParameterNew(s, 'cropLeft'), isFalse, reason: 'unchanged since 1.4.0'); + }); + + test('Noise Reduction: BM3D and KNLMeans in the method list', () { + final s = _load('noise_reduction.json'); + expect(s.methods.where(service.isMethodNew).map((m) => m.id).toSet(), {'bm3d', 'knlmeans'}); + expect(service.isMethodChoiceNew(s), isTrue); + expect(service.isSchemaNew(s), isTrue); + }); + + test('Frame Rate: RIFE in the method list', () { + final s = _load('frame_rate.json'); + expect(s.methods.where(service.isMethodNew).map((m) => m.id).toSet(), {'rife'}); + expect(service.isMethodChoiceNew(s), isTrue); + }); + + test('Descale is a new filter', () { + expect(service.isSchemaNew(_load('descale.json')), isTrue); + }); + + test('a filter with nothing new in 1.5.0 is not badged', () { + expect(service.isSchemaNew(_load('deband.json')), isFalse); + }); +} diff --git a/app/test/new_badges_widget_test.dart b/app/test/new_badges_widget_test.dart new file mode 100644 index 00000000..5e463363 --- /dev/null +++ b/app/test/new_badges_widget_test.dart @@ -0,0 +1,86 @@ +// The NEW badges as drawn by the generated panel, on the real Crop & Resize +// schema after an upgrade from 1.4.0: "Enable Upscale" badges because it +// reveals the new upscalers, "Upscale Method" because it holds them, and in the +// list exactly the three new options are badged. The labels only know about +// what they lead to through the panel's ParameterBadgeScope, so this is what +// catches that wiring breaking (each label would quietly fall back to its own +// stamp). +import 'dart:convert'; +import 'dart:io'; + +import 'package:flutter/material.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:package_info_plus/package_info_plus.dart'; +import 'package:provider/provider.dart'; +import 'package:shared_preferences/shared_preferences.dart'; +import 'package:vapourbox/models/dynamic_parameters.dart'; +import 'package:vapourbox/models/filter_schema.dart'; +import 'package:vapourbox/services/advanced_mode_service.dart'; +import 'package:vapourbox/services/whats_new_service.dart'; +import 'package:vapourbox/views/settings/dynamic_filter_panel.dart'; +import 'package:vapourbox/widgets/new_badge.dart'; + +void main() { + TestWidgetsFlutterBinding.ensureInitialized(); + + setUp(() async { + SharedPreferences.setMockInitialValues({ + 'lastRunAppVersion': '1.4.0', + 'lastSeenAppVersion': '1.4.0', + }); + PackageInfo.setMockInitialValues( + appName: 'VapourBox', packageName: 'app.vapourbox.vapourbox', + version: '1.5.0', buildNumber: '1', buildSignature: ''); + WhatsNewService.instance.resetForTesting(); + await WhatsNewService.instance.initialize(); + AdvancedModeService.instance.resetForTesting(); + await AdvancedModeService.instance.initialize(); + }); + + /// Whether the row holding [label] carries a NEW badge. + bool badged(WidgetTester tester, String label) { + expect(find.text(label), findsWidgets, reason: '"$label" is not on screen'); + final rows = find.ancestor(of: find.text(label), matching: find.byType(Row)); + return rows.evaluate().any((row) => find + .descendant(of: find.byElementPredicate((e) => e == row), matching: find.byType(NewBadge)) + .evaluate() + .isNotEmpty); + } + + testWidgets('Crop & Resize badges the way to the new upscalers', (tester) async { + final schema = FilterSchema.fromJson(jsonDecode( + File('assets/filters/core/crop_resize.json').readAsStringSync()) + as Map); + await tester.binding.setSurfaceSize(const Size(900, 4000)); + await tester.pumpWidget(ChangeNotifierProvider.value( + value: AdvancedModeService.instance, + child: MaterialApp( + home: Scaffold( + body: SingleChildScrollView( + child: DynamicFilterPanelCompact( + schema: schema, + params: DynamicParameters( + filterId: 'crop_resize', + enabled: true, + values: const {'useIntegerUpscale': true, 'upscaleMethod': 'nnedi3Rpow2'}, + enabledOptionalParams: const {'upscaleMethod', 'upscaleFactor'}, + ), + onChanged: (_) {}, + ), + ), + ), + ), + )); + await tester.pumpAndSettle(); + + expect(badged(tester, 'Enable Upscale'), isTrue, reason: 'it reveals the new upscalers'); + expect(badged(tester, 'Upscale Method'), isTrue, reason: 'it holds them'); + + await tester.tap(find.text('NNEDI3').last); + await tester.pumpAndSettle(); + for (final o in ['Real-ESRGAN', 'Real-CUGAN', 'AnimeJaNai (download, non-commercial)']) { + expect(badged(tester, o), isTrue, reason: o); + } + expect(badged(tester, 'EEDI3'), isFalse, reason: 'shipped in 1.4.0'); + }); +} diff --git a/app/test/whats_new_service_test.dart b/app/test/whats_new_service_test.dart index 23ca1557..54969667 100644 --- a/app/test/whats_new_service_test.dart +++ b/app/test/whats_new_service_test.dart @@ -183,4 +183,108 @@ void main() { false); }); }); + + // The rule everywhere: something new is badged, and so is every control on + // the way to it — a new option's dropdown, a new method's Method label, a + // switch that reveals a new control, a section holding one, the pass row. + group('badges lead to what is new', () { + ParameterDefinition p({String? since, Map? options, + Map? visibleWhen, bool hidden = false}) => + ParameterDefinition( + type: options != null ? ParameterType.enumType : ParameterType.boolean, + defaultValue: null, + options: options?.keys.toList(), + sinceAppVersion: since, + optionSinceAppVersion: options == null + ? null + : {for (final e in options.entries) if (e.value.isNotEmpty) e.key: e.value}, + ui: ParameterUiConfig(visibleWhen: visibleWhen, hidden: hidden), + ); + MethodDefinition m(String id, [String? since]) => MethodDefinition( + id: id, name: id, function: 'f', parameters: const [], sinceAppVersion: since); + FilterSchema schema(Map params, + {List methods = const []}) => + FilterSchema(id: 'x', version: '1.0.0', name: 'X', methods: methods, parameters: params); + + setUp(() async { + SharedPreferences.setMockInitialValues({ + 'lastRunAppVersion': '1.1.0', + 'lastSeenAppVersion': '1.1.0', + }); + setCurrentVersion('1.2.0'); + await service.initialize(); + }); + + test('a new option badges itself and its dropdown, not its siblings', () { + final method = p(options: {'old': '', 'fresh': '1.2.0'}); + final s = schema({'upscaleMethod': method}); + expect(service.isOptionNew(method, 'fresh'), isTrue); + expect(service.isOptionNew(method, 'old'), isFalse); + expect(service.isParameterNew(s, 'upscaleMethod'), isTrue); + expect(service.isSchemaNew(s), isTrue); + }); + + test('a switch badges for what it reveals, through every level', () { + // Enable Upscale -> Upscale Method (new option) -> a setting only that + // option shows; and a sibling switch revealing nothing new stays plain. + final s = schema({ + 'enable': p(), + 'method': p(options: {'old': '', 'fresh': '1.2.0'}, visibleWhen: {'enable': true}), + 'freshOnly': p(visibleWhen: {'method': ['fresh']}), + 'other': p(), + 'otherChild': p(visibleWhen: {'other': true}), + }); + expect(service.isParameterNew(s, 'enable'), isTrue); + expect(service.isParameterNew(s, 'other'), isFalse); + // Revealed by a new control, but new only through a level below. + final deep = schema({ + 'a': p(), + 'b': p(visibleWhen: {'a': true}), + 'c': p(since: '1.2.0', visibleWhen: {'b': true}), + }); + expect(service.isParameterNew(deep, 'a'), isTrue); + }); + + test('a hidden control reveals nothing and a cycle cannot recurse forever', () { + final s = schema({ + 'a': p(visibleWhen: {'b': true}), + 'b': p(visibleWhen: {'a': true}), + 'h': p(since: '1.2.0', hidden: true, visibleWhen: {'a': true}), + }); + expect(service.isParameterNew(s, 'a'), isFalse); + }); + + test('a new method badges the method list and the pass row', () { + final s = schema({}, methods: [m('old'), m('fresh', '1.2.0')]); + expect(service.isMethodNew(s.methods[1]), isTrue); + expect(service.isMethodNew(s.methods[0]), isFalse); + expect(service.isMethodChoiceNew(s), isTrue); + expect(service.isSchemaNew(s), isTrue); + // A new control that only some methods show badges the list too. + final gated = schema({'x': p(since: '1.2.0', visibleWhen: {'method': ['old']})}, + methods: [m('old'), m('older')]); + expect(service.isMethodChoiceNew(gated), isTrue); + }); + + test('a section badges when anything in it does', () { + final s = schema({'enable': p(), 'child': p(since: '1.2.0', visibleWhen: {'enable': true}), 'plain': p()}); + expect(service.isSectionNew(s, const UiSection(title: 'A', parameters: ['enable'])), isTrue); + expect(service.isSectionNew(s, const UiSection(title: 'B', parameters: ['plain'])), isFalse); + }); + + test('nothing badges once the update is no longer the latest', () async { + SharedPreferences.setMockInitialValues({ + 'lastRunAppVersion': '1.2.0', + 'lastSeenAppVersion': '1.2.0', + }); + service.resetForTesting(); + await service.initialize(); + final s = schema({ + 'enable': p(), + 'method': p(options: {'fresh': '1.2.0'}, visibleWhen: {'enable': true}), + }, methods: [m('a'), m('b', '1.2.0')]); + expect(service.isParameterNew(s, 'enable'), isFalse); + expect(service.isSchemaNew(s), isFalse); + }); + }); } diff --git a/docs/FILTER_SCHEMA.md b/docs/FILTER_SCHEMA.md index 51919b48..2a95ff6e 100644 --- a/docs/FILTER_SCHEMA.md +++ b/docs/FILTER_SCHEMA.md @@ -232,11 +232,31 @@ not the current one — and a "NEW" pill (`app/lib/widgets/new_badge.dart`) is drawn next to the pass in the pass list, or next to the parameter's control in the settings panel, whenever `sinceAppVersion` is newer than that. -A new parameter also badges its pass's row (`WhatsNewService.isSchemaNew`), -since passes start collapsed and its own badge is otherwise out of sight. -Parameters with `ui.hidden` don't count — there's no control to find. So -tagging only the parameter is enough; don't stamp the filter too unless the -filter itself is new. +**Stamp the newest thing, and the badges lead to it.** Three places take a stamp: + +| What is new | Stamp | Badged | +|---|---|---| +| a filter | top-level `sinceAppVersion` | its pass row | +| a parameter | the parameter's `sinceAppVersion` | its label | +| an option added to an existing enum | `"optionSinceAppVersion": {"