From fb0db23a58f127000141f067bdffe760a307e9a2 Mon Sep 17 00:00:00 2001 From: Andrew Stein Date: Sat, 19 Sep 2026 21:05:26 -0400 Subject: [PATCH] Add `Table::describe` Signed-off-by: Andrew Stein --- DEVELOPMENT.md | 206 ++- docs/md/explanation/view/advanced.md | 56 +- .../javascript/virtual_server/custom.md | 12 + .../md/how_to/python/virtual_server/custom.md | 11 +- .../src/ts/custom_elements/datagrid.ts | 4 +- .../src/ts/plugin/column_config_schema.ts | 64 +- .../test/js/transactional_restore.spec.ts | 227 +++ pnpm-workspace.yaml | 12 +- rust/perspective-client/build.rs | 6 +- rust/perspective-client/perspective.proto | 42 +- rust/perspective-client/src/rust/lib.rs | 7 +- rust/perspective-client/src/rust/table.rs | 205 ++- .../rust/virtual_server/generic_sql_model.rs | 89 +- .../generic_sql_model/table_make_view.rs | 2 +- .../virtual_server/generic_sql_model/tests.rs | 140 +- .../src/rust/virtual_server/handler.rs | 72 +- .../src/rust/virtual_server/mod.rs | 2 +- .../src/rust/virtual_server/server.rs | 124 +- .../src/rust/generic_sql_model.rs | 51 +- rust/perspective-js/src/rust/table.rs | 34 +- .../perspective-js/src/rust/virtual_server.rs | 66 +- .../src/ts/virtual_servers/clickhouse.ts | 101 +- .../src/ts/virtual_servers/duckdb.ts | 104 +- rust/perspective-js/test/js/describe.spec.js | 248 +++ .../test/js/duckdb/describe.spec.js | 159 ++ .../tests/table/test_view_expression.py | 57 + .../tests/virtual_servers/test_duckdb.py | 101 ++ .../tests/virtual_servers/test_polars.py | 28 + .../perspective/virtual_servers/__init__.py | 74 +- .../perspective/virtual_servers/clickhouse.py | 18 +- .../perspective/virtual_servers/duckdb.py | 18 +- .../perspective/virtual_servers/polars.py | 66 +- .../perspective/virtual_servers/postgres.py | 20 +- .../src/client/client_async.rs | 33 +- .../src/client/client_sync.rs | 8 + .../src/server/generic_sql_model.rs | 53 +- .../src/server/virtual_server_sync.rs | 42 +- .../patches/fix_arrow_libtool.patch | 3 +- .../cpp/perspective/src/cpp/server.cpp | 1135 +++++++------- .../cpp/perspective/src/cpp/view.cpp | 145 +- .../src/include/perspective/server.h | 15 + .../src/include/perspective/view.h | 8 + .../src/rust/agent/preamble.md | 1 + .../src/rust/agent/tools.rs | 16 +- .../column_settings_sidebar/style_tab.rs | 1 + .../src/rust/components/debug_panel.rs | 28 +- .../src/rust/components/plugin_tab.rs | 5 +- .../src/rust/components/viewer.rs | 5 +- .../src/rust/components/viewer/filters.rs | 14 +- .../src/rust/config/column_config_schema.rs | 38 + .../src/rust/config/options.rs | 10 +- .../src/rust/custom_elements/viewer.rs | 224 +-- rust/perspective-viewer/src/rust/js/plugin.rs | 2 +- .../src/rust/queries/plugin_column_styles.rs | 24 +- rust/perspective-viewer/src/rust/renderer.rs | 156 +- .../src/rust/renderer/plugin_config.rs | 943 ++++++++--- rust/perspective-viewer/src/rust/session.rs | 1393 ++++++++++++----- .../src/rust/session/metadata.rs | 20 +- .../src/rust/session/op_queue.rs | 393 +++++ .../src/rust/session/panel_state.rs | 463 ++++++ .../src/rust/tasks/apply_global_filters.rs | 87 +- .../src/rust/tasks/auto_pause.rs | 31 +- .../src/rust/tasks/create_panel.rs | 34 +- rust/perspective-viewer/src/rust/tasks/mod.rs | 4 +- .../src/rust/tasks/pipeline.rs | 232 ++- .../src/rust/tasks/reset_all.rs | 51 +- .../src/rust/tasks/restore_and_render.rs | 107 -- .../src/rust/tasks/restore_panel.rs | 323 ++-- .../src/rust/tasks/send_column_config.rs | 75 +- .../src/rust/tasks/send_plugin_config.rs | 67 +- .../src/rust/tasks/set_edit_mode.rs | 50 +- .../src/rust/tasks/table_lifecycle.rs | 99 +- .../src/rust/tasks/transactional_restore.rs | 402 +++++ rust/perspective-viewer/src/rust/workspace.rs | 56 +- rust/perspective-viewer/src/ts/plugin.ts | 5 +- .../js/dragdrop/drop_column_settings.spec.ts | 20 + .../test/js/multi_panel/global_filter.spec.ts | 131 ++ .../js/multi_panel/load_restore_race.spec.ts | 16 +- .../test/js/status_indicator/updating.spec.ts | 8 +- .../test/js/viewer_api/op_queue.spec.ts | 235 +++ .../js/viewer_api/table_lifecycle.spec.ts | 131 +- 81 files changed, 7511 insertions(+), 2257 deletions(-) create mode 100644 packages/viewer-datagrid/test/js/transactional_restore.spec.ts create mode 100644 rust/perspective-js/test/js/describe.spec.js create mode 100644 rust/perspective-js/test/js/duckdb/describe.spec.js create mode 100644 rust/perspective-viewer/src/rust/session/op_queue.rs create mode 100644 rust/perspective-viewer/src/rust/session/panel_state.rs delete mode 100644 rust/perspective-viewer/src/rust/tasks/restore_and_render.rs create mode 100644 rust/perspective-viewer/src/rust/tasks/transactional_restore.rs create mode 100644 rust/perspective-viewer/test/js/viewer_api/op_queue.spec.ts diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 807ad8e211..4b55a60283 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -12,20 +12,48 @@ might not work the way you're used to! Perspective is organized as a [monorepo](https://github.com/babel/babel/blob/master/doc/design/monorepo.md), -and uses [lerna](https://lernajs.io/) to manage dependencies. - -This guide provides instructions for both the JavaScript and Python libraries. -To switch your development toolchain between the two, use `pnpm run setup`. Once -the setup script has been run, common commands like `pnpm run build` and -`pnpm run test` automatically call the correct build and test tools. +and uses [pnpm workspaces](https://pnpm.io/workspaces) to manage dependencies. +All commands in this guide are run from the repository root. + +| Path | Contents | +| ------------------------------------------------------------- | -------------------------------------------------------------------- | +| `rust/perspective-server` | The C++ engine, compiled natively and to WebAssembly via Emscripten | +| `rust/perspective-client` | Rust client, `perspective.proto` and the Virtual Server framework | +| `rust/perspective-js` | `@perspective-dev/client`, the JavaScript/WebAssembly bindings | +| `rust/perspective-python` | `perspective-python`, the [PyO3](https://pyo3.rs) bindings | +| `rust/perspective-viewer` | `@perspective-dev/viewer`, the `` Custom Element | +| `rust/perspective` | The `perspective` Rust crate | +| `packages/viewer-datagrid`, `packages/viewer-charts` | Viewer plugins | +| `packages/react`, `packages/jupyterlab`, `packages/anywidget` | Framework and notebook integrations | +| `tools/scripts`, `tools/test`, `tools/bench` | Build scripts, the shared test harness and the benchmark suite | +| `examples`, `docs` | Example projects and the documentation site | + +This guide provides instructions for the JavaScript, Python and Rust libraries. +To choose which packages your development toolchain builds and tests, use +`pnpm run setup`. Once the setup script has been run, common commands like +`pnpm run build` and `pnpm run test` automatically call the correct build and +test tools for the selected packages. ### System Dependencies `Perspective.js` and `perspective-python` **require** the following system dependencies to be installed: +- [Node.js](https://nodejs.org/) (version 22 is what CI uses) +- [pnpm](https://pnpm.io/) +- [Rust](https://rustup.rs/) via `rustup`. The pinned nightly toolchain and + WebAssembly targets in `rust-toolchain.toml` are installed automatically. - [CMake](https://cmake.org/) (version 3.29.5 or higher) -- [pnpm](https://pnpm.io/). +- A C++17 compiler for native builds (`perspective-python` and the Rust crate). + LLVM 17 is the pinned version, which `pnpm run install_llvm` will download to + `.llvm/`. + +Running `pnpm install` additionally downloads the pinned versions of +[Emscripten](https://emscripten.org/) and +[Binaryen](https://github.com/WebAssembly/binaryen) specified in `package.json`, +and the Chromium build used by [Playwright](https://playwright.dev/). Boost and +the other C++ dependencies are downloaded by CMake at build time, and do not +need to be installed. **_This list may be non-exhaustive depending on your OS/environment; please open a thread in @@ -51,18 +79,44 @@ a `.perspectiverc` via a short survey. This can be later re-configured via pnpm run setup ``` +`.perspectiverc` is a plain `KEY=value` file, and any of its values can be +overridden per-command from the environment. `PACKAGE` is a comma-separated list +of the package names shown by `pnpm run setup`, e.g. to build just the engine +and JavaScript client: + +```bash +PACKAGE=server,client pnpm run build +``` + +Note that `PACKAGE` is a filter, not a dependency graph. Packages which are not +selected are not rebuilt, so e.g. a change to the C++ engine in +`rust/perspective-server` will not be reflected in `@perspective-dev/client` +unless `server` is also selected. + +Other useful options: + +| Variable | Effect | +| --------------------- | ---------------------------------------------------------- | +| `PSP_DEBUG=1` | Debug build | +| `PSP_BUILD_VERBOSE=1` | Verbose C++ build output | +| `PSP_NUM_CPUS=` | Limit C++ build parallelism | +| `PSP_WASM64=1` | Also build the `wasm64` (Memory64) engine | +| `PSP_BUILD_WHEEL=1` | Build a `perspective-python` wheel to `rust/target/wheels` | +| `PSP_DOCKER=1` | Build inside the Docker build environment | + If everything is successful, you should be able to run any of the `examples/` -packages, e.g. `examples/blocks` like so: +packages, e.g. `examples/esbuild-example` like so: ```bash -pnpm run start blocks +pnpm run start esbuild-example ``` +To remove build artifacts, use `pnpm run clean`. + ## `Perspective.js` To build the JavaScript library, which includes WebAssembly compilation, -[Emscripten](https://github.com/kripken/emscripten) and its prerequisites are -required. +[Emscripten](https://emscripten.org/) and its prerequisites are required. `Perspective.js` specifies its Emscripten version dependency in `package.json`, and the correct version of Emscripten will be installed with other JS @@ -84,12 +138,15 @@ Deviating from this specific version of Emscripten specified in the project's `package.json` can introduce various errors that are extremely difficult to debug. -To install a specific version of Emscripten (e.g. `2.0.6`): +To install a specific version of Emscripten (e.g. `4.0.9`): ```bash -./emsdk install 2.0.6 +./emsdk install 4.0.9 ``` +Set `PSP_SKIP_EMSDK_INSTALL=1` to prevent `pnpm install` from downloading the +bundled version. + --- ## `perspective-python` @@ -102,27 +159,43 @@ of python, e.g. pip install -r rust/perspective-python/requirements.txt ``` +`pnpm run build` will then compile the extension and install it into your active +Python environment in development mode via +[`maturin develop`](https://www.maturin.rs/). It is strongly recommended to do +this within a virtual environment. + `perspective-python` supports Python 3.11 and upwards. +To build for [Pyodide](https://pyodide.org/), select +`perspective-python (pyodide)` in `pnpm run setup` and install the pinned +Pyodide distribution with `pnpm run install_pyodide`. + ### `perspective-jupyterlab` -To install the Jupyterlab/Jupyter Notebook plugins from your local working -directory, simply install `python/perspective` with `pip` as you might normally -do. +The JupyterLab extension is built by the `jupyterlab` package, which copies the +resulting labextension into the `perspective-python` package's data directory. +To install it from your local working directory, build both packages as a wheel +and install the wheel with `pip`: ```bash -# builds labextension to the perspective-python python package root directory -PACKAGE=perspective-jupyterlab pnpm run build -# editable install of the python package -pnpm -F @perspective-dev/python develop:maturin -# set up symlink of our labextension to jupyter share directory -# this directory's path is in the output of `jupyter labextension list` -pnpm -F @perspective-dev/python develop:labextension +# builds the labextension, then a wheel which bundles it +PACKAGE=jupyterlab,python PSP_BUILD_WHEEL=1 pnpm run build +pip install --force-reinstall rust/target/wheels/perspective_python-*.whl ``` -Afterwards, you should see it listed as a "local extension" when you run -`jupyter labextension list` and as a normal extension when you run -`jupyter nbextension list`. +Afterwards, you should see `@perspective-dev/jupyterlab` listed when you run +`jupyter labextension list`. + +## `perspective` (Rust) + +To build the Rust crate, select `perspective (rust)` in `pnpm run setup`. + +The root `.cargo/config.toml` shares one `target-dir` (`rust/target`) between +the native and WebAssembly builds. When invoking `cargo` directly, always pass +an explicit `--target` (e.g. `--target=wasm32-unknown-unknown` for +`perspective-viewer` and `perspective-js`), otherwise `cargo` will fingerprint +shared host artifacts differently than `pnpm run build` does, and each will +invalidate the other's cache. --- @@ -137,7 +210,7 @@ brew install cmake llvm@17 brew link llvm@17 # optional, see below ``` -On M1 (Apple Silicon) systems, make sure your brew-installed dependencies are in +On Apple Silicon systems, make sure your brew-installed dependencies are in `/opt/homebrew` (the default location), and that `/opt/homebrew/bin` is on the `PATH`. @@ -168,28 +241,28 @@ prerequisite tools. ### Ubuntu/Debian -On Ubuntu, CMake will mistakenly resolve the system headers in `/usr/include` -rather than the emscripten supplied versions. You can resolve this by moving -`boost` dependencies to somewhere other than `/usr/include` - into Perspective's -own `src` dir (as per -[here](http://vclf.blogspot.com/2014/08/emscripten-linking-to-boost-libraries.html)). +Install system dependencies through `apt`: ```bash -apt-get install libboost-all-dev -cp -r /usr/include/boost ./packages/perspective/src/include/ +apt-get install build-essential cmake ``` +Boost is downloaded by CMake at build time; a system `libboost` is not required. + --- ## Test -You can run the test suite simply with the standard NPM command, which will both -build the test suite for every package and run them. +You can run the test suite for the packages selected in `.perspectiverc` with +the standard NPM command. ```bash pnpm run test ``` +The test suite runs against the artifacts of the last `pnpm run build`, and does +not rebuild them; remember to re-run the build after making a change. + ### JavaScript The JavaScript test suite is composed of two sections: a Node.js test, which @@ -197,10 +270,65 @@ asserts behavior of the `@perspective-dev/client` library, and a suite of [Playwright](https://playwright.dev/) tests, which assert the behavior of the rest of the UI facing packages. +`PACKAGE` selects which packages' suites run, and extra arguments are forwarded +to Playwright, so to run a single spec or test: + +```bash +PACKAGE=viewer-datagrid pnpm run test column_style.spec +PACKAGE=client pnpm run test -g "to_arrow" +``` + +Each package is a Playwright project named `-desktop-chrome` (or +`-node` for the Node.js suites), which can be passed as `--project` to +narrow a run when several packages are selected. + +Set `PSP_HEADED=1` to watch the browser tests run. The JupyterLab integration +tests are run with `PACKAGE=jupyterlab pnpm run test --jupyter`. + +Many UI tests compare against screenshot and DOM snapshots, which live in +`tools/test/dist/snapshots` and are not checked in to this repository. CI +fetches them from a separate snapshots repository. To regenerate +snapshots locally after an intentional rendering change, or to generate them +for the first time from a known-passing (in CI) build you've checked out: + ```bash pnpm run test --update-snapshots ``` +Locally regenerated snapshots only affect your machine; a pull request which +changes rendering also needs its snapshots veriied and published to the +snapshots repository by a maintainer. + +### Python + +With `python` selected, `pnpm run test` runs the `pytest` suite; extra arguments +are forwarded to `pytest`. + +### Rust + +With `rust` selected, `pnpm run test` runs `cargo test` for the `perspective` +and `perspective-client` crates. + +## Lint + +```bash +pnpm run lint +pnpm run fix +``` + +`lint` checks license headers, `eslint`, `prettier`, `clippy` and `rustfmt` (and +`ruff` when `perspective-python` is selected), and is run as a pre-push hook. +`fix` applies the automatic fixes. + +## Docs + +```bash +pnpm run docs +``` + +Documentation sources live in `docs/md`. This runs `cargo doc`, then the `docs` +script of each selected package. + ### Troubleshooting installation from source If you are installing from a source distribution (sdist), make sure you have the @@ -215,7 +343,7 @@ pip install -vv perspective-python The most common culprits are: - CMake version is too old -- Boost headers are missing or too old +- No C++17 compiler is available --- @@ -223,7 +351,9 @@ The most common culprits are: You can generate benchmarks specific to your machine's OS and CPU architecture with Perspective's benchmark suite, which will host a live dashboard at -http://localhost:8080 as well as output a result `benchmark.arrow` file. +http://localhost:8080 as well as output a result `.arrow` file to +`tools/bench/dist`. The suite which runs is chosen by the selected packages: +`viewer-charts`, `client` or `python`. ```bash pnpm run bench diff --git a/docs/md/explanation/view/advanced.md b/docs/md/explanation/view/advanced.md index a757223886..a4063af1ec 100644 --- a/docs/md/explanation/view/advanced.md +++ b/docs/md/explanation/view/advanced.md @@ -79,11 +79,61 @@ min_val, max_val = view.get_min_max("Sales") +## Describing a View Config + +`Table::describe` validates a complete view config against a table and +reports the schema a `View` built from it would have - without creating one. +`describe` reports a `view_schema` if and only if `Table::view` with the same +config would succeed, and the two schemas are equal. It costs no engine +resources, so it is the right way to check a config before applying it. + +
+ +```javascript +const verdict = await table.describe({ + columns: ["Sales", "margin"], + group_by: ["Region"], + expressions: { margin: '"Profit" / "Sales"' }, + aggregates: { margin: "avg" }, +}); + +if ("view_schema" in verdict) { + // { expression_schema: { margin: "float" }, + // view_schema: { Sales: "float", margin: "float" } } +} else if ("expression_errors" in verdict) { + // { expression_schema: {...}, expression_errors: { margin: { error_message, line, column } } } +} else { + // { config_error: "Invalid column 'Sales' found in View columns." } +} +``` + +
+
+ +```python +verdict = table.describe( + columns=["Sales", "margin"], + group_by=["Region"], + expressions={"margin": '"Profit" / "Sales"'}, + aggregates={"margin": "avg"}, +) + +if "view_schema" in verdict: + ... +elif "expression_errors" in verdict: + ... +else: + verdict["config_error"] +``` + +
+ ## Expression Validation -Before creating a `View` with expressions, you can validate them against the -table's schema using `Table::validate_expressions`. This returns information -about which expressions are valid and their inferred types: +`Table::validate_expressions` is a specialization of `Table::describe` over +a config that selects no columns, so only the expressions are checked. It +returns which expressions are valid and their inferred types, plus an +`expression_alias` map echoing the request:
diff --git a/docs/md/how_to/javascript/virtual_server/custom.md b/docs/md/how_to/javascript/virtual_server/custom.md index 38e7a3a1c2..603b75f578 100644 --- a/docs/md/how_to/javascript/virtual_server/custom.md +++ b/docs/md/how_to/javascript/virtual_server/custom.md @@ -13,6 +13,7 @@ import perspective from "@perspective-dev/client"; import type { VirtualServerHandler, ColumnType, + TableDescription, ViewConfig, ViewWindow, VirtualDataSlice, @@ -41,6 +42,17 @@ const handler = { // for later data retrieval. }, + async tableDescribe( + tableId: string, + config: ViewConfig, + ): Promise { + // Validate `config` against `tableId` and report the schema the view + // would have, WITHOUT creating it: `{ expression_schema, view_schema }` + // when valid, `{ expression_schema, expression_errors }` when an + // expression is invalid, or `{ config_error }` otherwise. + return { expression_schema: {}, view_schema: { name: "string", price: "float" } }; + }, + async viewDelete(viewId: string): Promise { // Clean up resources for this view }, diff --git a/docs/md/how_to/python/virtual_server/custom.md b/docs/md/how_to/python/virtual_server/custom.md index d24e273a98..be3b711e71 100644 --- a/docs/md/how_to/python/virtual_server/custom.md +++ b/docs/md/how_to/python/virtual_server/custom.md @@ -51,6 +51,16 @@ class MyHandler(VirtualServerHandler): # for later data retrieval. pass + def table_describe(self, table_name, config): + # Validate `config` against `table_name` and report the schema the + # temporary table would have, WITHOUT creating it. Return + # {"expression_schema", "view_schema"} when valid, + # {"expression_schema", "expression_errors"} when an expression is + # invalid, or {"config_error"} otherwise. A model with no cheaper + # answer may `return describe_via_make_view(self, table_name, config)`, + # which builds and drops a real temporary table - at that cost. + pass + def view_delete(self, view_name): # Clean up resources for this view. The UI does this automatically, # and can recover if a view dies early. @@ -70,7 +80,6 @@ class MyHandler(VirtualServerHandler): | `get_features()` | `columns` only | Which UI controls to enable — see [Features declaration](../../../explanation/virtual_servers.md#features-declaration) | | `view_schema(view_name, config)` | `table_schema` | Schema of a temporary table, when it differs from its source | | `view_size(view_name)` | `table_size` | Row count of a temporary table, when it differs from its source | -| `table_validate_expression(view_name, expression)` | allow all | Type-check an expression column; enabled by `"expressions"` in `get_features` | | `view_get_min_max(view_name, column_name, config)` | unsupported | Column bounds as a `(min, max)` tuple — required for gradient and sparkbar column styles | ## The session factory diff --git a/packages/viewer-datagrid/src/ts/custom_elements/datagrid.ts b/packages/viewer-datagrid/src/ts/custom_elements/datagrid.ts index 9982652f05..677a662697 100644 --- a/packages/viewer-datagrid/src/ts/custom_elements/datagrid.ts +++ b/packages/viewer-datagrid/src/ts/custom_elements/datagrid.ts @@ -236,7 +236,7 @@ export class HTMLPerspectiveViewerDatagridPluginElement split_by?: string[]; group_rollup_mode?: string; }, - column_stats?: { abs_max: number }, + plugin_config?: Record | null, ): ColumnConfigSchema { return column_config_schema.call( this, @@ -245,7 +245,7 @@ export class HTMLPerspectiveViewerDatagridPluginElement column_name, current_value, viewer_config, - column_stats, + plugin_config, ); } diff --git a/packages/viewer-datagrid/src/ts/plugin/column_config_schema.ts b/packages/viewer-datagrid/src/ts/plugin/column_config_schema.ts index 0099b614da..3fdfb9dbc9 100644 --- a/packages/viewer-datagrid/src/ts/plugin/column_config_schema.ts +++ b/packages/viewer-datagrid/src/ts/plugin/column_config_schema.ts @@ -12,7 +12,9 @@ import type { ColumnType } from "@perspective-dev/client"; import { colorsToCss, rgbToHex, stopsToCss } from "../color_utils.js"; -import { measure_px } from "./plugin_config_schema.js"; +import { readThemeStyle } from "../model/create.js"; +import { measure_px, parse_align } from "./plugin_config_schema.js"; +import { positive_px } from "./restore.js"; import { bg_modes_for, default_bg_mode, @@ -42,10 +44,6 @@ export interface ColumnConfigSchema { * Plugin schema for the Datagrid column-settings sidebar. Returns the * controls the viewer should render in the Style tab for a given column. */ -interface ColumnStats { - abs_max?: number; -} - export default function column_config_schema( this: DatagridPluginElement, type: ColumnType, @@ -53,8 +51,30 @@ export default function column_config_schema( _column_name: string, current_value: Record | null, viewer_config?: ViewerConfigLike, - column_stats?: ColumnStats, + plugin_config?: Record | null, ): ColumnConfigSchema { + const grid = + plugin_config === undefined || plugin_config === null + ? { + font_family: this._font_family, + font_size: this._font_size, + bold: this._bold, + italic: this._italic, + align: this._align, + word_wrap: this._word_wrap, + } + : { + font_family: + typeof plugin_config.font_family === "string" + ? plugin_config.font_family + : undefined, + font_size: positive_px(plugin_config.font_size), + bold: plugin_config.bold === true, + italic: plugin_config.italic === true, + align: parse_align(plugin_config.align), + word_wrap: plugin_config.word_wrap === true, + }; + const fields: ControlSpec[] = []; const group: ControlSpec & { fields: ControlSpec[] } = { kind: "Group", @@ -88,33 +108,33 @@ export default function column_config_schema( { kind: "Font", key: "font_family" satisfies keyof ColumnConfig, - default: this._font_family ?? "inherit", + default: grid.font_family ?? "inherit", size: { key: "font_size" satisfies keyof ColumnConfig, default: - this._font_size ?? measure_px(this, "font-size", 12), + grid.font_size ?? measure_px(this, "font-size", 12), min: 4, max: 96, step: 1, }, bold: { key: "bold" satisfies keyof ColumnConfig, - default: this._bold, + default: grid.bold, }, italic: { key: "italic" satisfies keyof ColumnConfig, - default: this._italic, + default: grid.italic, }, }, { kind: "Alignment", key: "align" satisfies keyof ColumnConfig, - ...(this._align !== undefined ? { default: this._align } : {}), + ...(grid.align !== undefined ? { default: grid.align } : {}), }, { kind: "Bool", key: "word_wrap" satisfies keyof ColumnConfig, - default: this._word_wrap, + default: grid.word_wrap, }, ], }); @@ -132,9 +152,9 @@ export default function column_config_schema( const color_fields: ControlSpec[] = [ mode_spec("fg_mode", fg_modes, default_fg_mode(type)), - ...value_specs.call(this, type, "fg", fg_mode, column_stats), + ...value_specs.call(this, type, "fg", fg_mode), mode_spec("bg_mode", bg_modes, default_bg_mode(type)), - ...value_specs.call(this, type, "bg", bg_mode, column_stats), + ...value_specs.call(this, type, "bg", bg_mode), ]; fields.push({ kind: "Group", key: "color", fields: color_fields }); @@ -191,16 +211,17 @@ function value_specs( type: ColumnType, side: "fg" | "bg", mode: FgMode | BgMode, - column_stats: ColumnStats | undefined, ): ControlSpec[] { if (mode === "disabled") { return []; } + const theme = this.model ?? readThemeStyle(this.regular_table); + const key = `${side}_color` satisfies keyof ColumnConfig; if (type === "integer" || type === "float") { - const pos = this.model![`_pos_${side}_color`][0]; - const neg = this.model![`_neg_${side}_color`][0]; + const pos = theme[`_pos_${side}_color`][0]; + const neg = theme[`_neg_${side}_color`][0]; const stops: ControlSpec = mode === "gradient" || mode === "pulse" ? { @@ -210,7 +231,7 @@ function value_specs( { color: neg, offset: 0 }, { color: rgbToHex( - this.model!._plugin_background as [ + theme._plugin_background as [ number, number, number, @@ -241,7 +262,8 @@ function value_specs( kind: "Number", key: `${side}_gradient` satisfies keyof ColumnConfig, include: true, - default: column_stats?.abs_max ?? 0, + default: 0, + default_stat: "abs_max", }, ] : [stops]; @@ -250,10 +272,10 @@ function value_specs( { kind: "Palette", key, - default: colorsToCss(this.model!._series_palette), + default: colorsToCss(theme._series_palette), }, ]; } else { - return [{ kind: "Color", key, default: this.model!._color[0] }]; + return [{ kind: "Color", key, default: theme._color[0] }]; } } diff --git a/packages/viewer-datagrid/test/js/transactional_restore.spec.ts b/packages/viewer-datagrid/test/js/transactional_restore.spec.ts new file mode 100644 index 0000000000..72b8d9daa9 --- /dev/null +++ b/packages/viewer-datagrid/test/js/transactional_restore.spec.ts @@ -0,0 +1,227 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +import { expect, test } from "@perspective-dev/test"; +import type { Page } from "@playwright/test"; + +const BAD_COLUMNS_CONFIG = { Sales: { fg_mode: "not-a-mode" } }; + +async function attempt( + page: Page, + config: Record, + options?: Record, +) { + return page.evaluate( + async ({ config, options }) => { + const viewer = document.querySelector("perspective-viewer")! as any; + const grid = document.querySelector( + "perspective-viewer-datagrid", + ) as any; + + let renders = 0; + const restore_fns: Array<() => void> = []; + if (grid) { + for (const name of ["draw", "update"]) { + const original = grid[name]; + grid[name] = function (...args: unknown[]) { + renders++; + return original.apply(this, args); + }; + + restore_fns.push(() => (grid[name] = original)); + } + } + + await viewer.flush(); + const before = await viewer.save(); + let error: string | null = null; + try { + await viewer.restore(config, options); + } catch (e: any) { + error = String(e?.message ?? e); + } + + await viewer.flush(); + const after = await viewer.save(); + restore_fns.forEach((f) => f()); + return { before, after, error, renders }; + }, + { config, options }, + ); +} + +test.describe("Transactional restore", function () { + test.beforeEach(async ({ page }) => { + await page.goto("/tools/test/src/html/basic-test.html"); + await page.evaluate(async () => { + while (!(window as any)["__TEST_PERSPECTIVE_READY__"]) { + await new Promise((x) => setTimeout(x, 10)); + } + }); + + await page.evaluate(async () => { + const viewer = document.querySelector("perspective-viewer")! as any; + await viewer.restore({ + plugin: "Datagrid", + columns: ["Sales", "Profit"], + group_by: ["Region"], + title: "Before", + }); + + await viewer.flush(); + }); + }); + + test("an invalid columns_config rejects and changes nothing", async ({ + page, + }) => { + const result = await attempt(page, { + columns_config: BAD_COLUMNS_CONFIG, + }); + + expect(result.error).toContain("columns_config"); + expect(result.after).toEqual(result.before); + expect(result.renders).toBe(0); + }); + + test("a valid view config beside an invalid columns_config is not applied", async ({ + page, + }) => { + const result = await attempt(page, { + group_by: ["State"], + split_by: ["Category"], + columns_config: BAD_COLUMNS_CONFIG, + }); + + expect(result.error).not.toBeNull(); + expect(result.after).toEqual(result.before); + expect(result.renders).toBe(0); + }); + + test("a valid plugin_config beside an invalid columns_config is not applied", async ({ + page, + }) => { + const result = await attempt(page, { + plugin_config: { edit_mode: "EDIT" }, + columns_config: BAD_COLUMNS_CONFIG, + }); + + expect(result.error).not.toBeNull(); + expect(result.after).toEqual(result.before); + }); + + test("a title beside an invalid columns_config is not applied", async ({ + page, + }) => { + const result = await attempt(page, { + title: "After", + columns_config: BAD_COLUMNS_CONFIG, + }); + + expect(result.error).not.toBeNull(); + expect(result.after.title).toBe("Before"); + }); + + test("an invalid expression rejects and changes nothing", async ({ + page, + }) => { + const result = await attempt(page, { + expressions: { broken: '"Sales" +' }, + columns: ["broken"], + }); + + expect(result.error).not.toBeNull(); + expect(result.after).toEqual(result.before); + expect(result.renders).toBe(0); + }); + + test("an unknown column rejects and changes nothing", async ({ page }) => { + const result = await attempt(page, { group_by: ["Not A Column"] }); + expect(result.error).toContain("Not A Column"); + expect(result.after).toEqual(result.before); + expect(result.renders).toBe(0); + }); + + test("an unknown plugin name rejects and changes nothing", async ({ + page, + }) => { + const result = await attempt(page, { + plugin: "Not A Plugin", + group_by: ["State"], + }); + + expect(result.error).toContain('Unknown plugin "Not A Plugin"'); + expect(result.after).toEqual(result.before); + expect(result.renders).toBe(0); + }); + + test("a plugin swap beside an invalid columns_config does not swap", async ({ + page, + }) => { + await page.evaluate(async () => { + const viewer = document.querySelector("perspective-viewer")! as any; + await viewer.restore({ plugin: "Y Bar" }); + await viewer.flush(); + }); + + const result = await attempt(page, { + plugin: "Datagrid", + columns_config: BAD_COLUMNS_CONFIG, + }); + + expect(result.error).toContain("columns_config"); + expect(result.before.plugin).toBe("Y Bar"); + expect(result.after.plugin).toBe("Y Bar"); + expect(result.after).toEqual(result.before); + }); + + test("a rejected restore leaves the panel usable without suppress_errors", async ({ + page, + }) => { + const rejected = await attempt(page, { + columns_config: BAD_COLUMNS_CONFIG, + }); + + expect(rejected.error).not.toBeNull(); + const accepted = await attempt(page, { group_by: ["State"] }); + expect(accepted.error).toBeNull(); + expect(accepted.after.group_by).toEqual(["State"]); + }); + + test("a rejected restore behaves the same with suppress_errors", async ({ + page, + }) => { + const result = await attempt( + page, + { group_by: ["State"], columns_config: BAD_COLUMNS_CONFIG }, + { suppress_errors: true }, + ); + + expect(result.error).not.toBeNull(); + expect(result.after).toEqual(result.before); + }); + + test("a valid restore applies every part together", async ({ page }) => { + const result = await attempt(page, { + group_by: ["State"], + title: "After", + plugin_config: { edit_mode: "EDIT" }, + columns_config: { Sales: { fg_mode: "bar" } }, + }); + + expect(result.error).toBeNull(); + expect(result.after.group_by).toEqual(["State"]); + expect(result.after.title).toBe("After"); + expect(result.after.plugin_config.edit_mode).toBe("EDIT"); + expect(result.after.columns_config.Sales.fg_mode).toBe("bar"); + }); +}); diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index fe1cd4380f..28dcfed804 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -21,12 +21,12 @@ packages: - "docs" allowBuilds: - '@fortawesome/fontawesome-free': true - aws-sdk: true - core-js: true - esbuild: true - microtime: true - puppeteer: true + "@fortawesome/fontawesome-free": true + aws-sdk: true + core-js: true + esbuild: true + microtime: true + puppeteer: false catalog: # Dependencies diff --git a/rust/perspective-client/build.rs b/rust/perspective-client/build.rs index 5b7246d1fc..d19d6e33b1 100644 --- a/rust/perspective-client/build.rs +++ b/rust/perspective-client/build.rs @@ -79,12 +79,14 @@ fn prost_build() -> Result<()> { .type_attribute(".", "#[derive(serde::Serialize)]") .type_attribute("ViewPort", "#[derive(serde::Deserialize)]") .type_attribute("ViewDimensionsResp", "#[derive(serde::Deserialize)]") - .type_attribute("TableValidateExprResp", "#[derive(serde::Deserialize)]") + .type_attribute("TableDescribeResp", "#[derive(serde::Deserialize)]") + .type_attribute("TableDescribeResp.result", "#[derive(serde::Deserialize)]") + .type_attribute("ViewDescription", "#[derive(serde::Deserialize)]") .type_attribute( "ColumnType", "#[derive(serde::Deserialize)] #[serde(rename_all = \"snake_case\")]", ) - .type_attribute("ExprValidationError", "#[derive(serde::Deserialize)]") + .type_attribute("ExpressionError", "#[derive(serde::Deserialize)]") .compile_protos(&[proto_file], &[include_path]) .unwrap(); diff --git a/rust/perspective-client/perspective.proto b/rust/perspective-client/perspective.proto index c13a2032ca..2d1d958432 100644 --- a/rust/perspective-client/perspective.proto +++ b/rust/perspective-client/perspective.proto @@ -116,6 +116,7 @@ enum SortOp { message Request { uint32 msg_id = 1; string entity_id = 2; + reserved 9; // `TableValidateExprReq`, replaced by `TableDescribeReq` oneof client_req { // Minimum Virtual API (theoretical). GetFeaturesReq get_features_req = 3; @@ -125,7 +126,7 @@ message Request { TableMakeViewReq table_make_view_req = 6; TableSchemaReq table_schema_req = 7; TableSizeReq table_size_req = 8; - TableValidateExprReq table_validate_expr_req = 9; + TableDescribeReq table_describe_req = 41; ViewColumnPathsReq view_column_paths_req = 10; ViewDeleteReq view_delete_req = 11; ViewDimensionsReq view_dimensions_req = 12; @@ -167,6 +168,7 @@ message Request { message Response { uint32 msg_id = 1; string entity_id = 2; + reserved 9; // `TableValidateExprResp`, replaced by `TableDescribeResp` oneof client_resp { GetFeaturesResp get_features_resp = 3; GetHostedTablesResp get_hosted_tables_resp = 4; @@ -175,7 +177,7 @@ message Response { TableMakeViewResp table_make_view_resp = 6; TableSchemaResp table_schema_resp = 7; TableSizeResp table_size_resp = 8; - TableValidateExprResp table_validate_expr_resp = 9; + TableDescribeResp table_describe_resp = 41; ViewColumnPathsResp view_column_paths_resp = 10; ViewDeleteResp view_delete_resp = 11; ViewDimensionsResp view_dimensions_resp = 12; @@ -307,21 +309,35 @@ message TableSchemaResp { Schema schema = 1; } -// `Table::validate_expressions` -// TODO: This should be just `validate()` -message TableValidateExprReq { - map column_to_expr = 1; +// `Table::describe` - validate a complete `ViewConfig` against this table and +// report its result schema, without creating a `View`. +message TableDescribeReq { + ViewConfig config = 1; } -message TableValidateExprResp { +message TableDescribeResp { + // Every VALID expression's type, whether or not the config as a whole is + // valid. map expression_schema = 1; - map errors = 2; - map expression_alias = 3; - message ExprValidationError { - string error_message = 1; - uint32 line = 2; - uint32 column = 3; + map expression_errors = 2; + oneof result { + // Present IFF `expression_errors` is empty and the whole config is + // valid. + ViewDescription view = 3; + + // The first non-expression config error, when the expressions are all + // valid but the config is not. + string config_error = 4; } } +message ViewDescription { + // Equal to `View::schema()` of a `View` built from the same config. + map schema = 1; +} +message ExpressionError { + string error_message = 1; + uint32 line = 2; + uint32 column = 3; +} // `Table::view` message TableMakeViewReq { diff --git a/rust/perspective-client/src/rust/lib.rs b/rust/perspective-client/src/rust/lib.rs index cde1b27f63..fa59b57144 100644 --- a/rust/perspective-client/src/rust/lib.rs +++ b/rust/perspective-client/src/rust/lib.rs @@ -55,8 +55,8 @@ use crate::proto::HostedTable; pub use crate::proto::JoinType; pub use crate::session::{ProxySession, Session}; pub use crate::table::{ - DeleteOptions, ExprValidationResult, JoinOptions, Table, TableInitOptions, TableReadFormat, - UpdateOptions, + DeleteOptions, DescribeError, DescribeVerdict, Description, ExprValidationResult, JoinOptions, + Table, TableInitOptions, TableReadFormat, UpdateOptions, }; pub use crate::table_data::{TableData, UpdateData}; pub use crate::table_ref::TableRef; @@ -65,7 +65,7 @@ pub use crate::view::{ }; pub type ClientError = utils::ClientError; -pub type ExprValidationError = crate::proto::table_validate_expr_resp::ExprValidationError; +pub type ExprValidationError = crate::proto::ExpressionError; #[doc(hidden)] pub mod vendor { @@ -99,6 +99,7 @@ macro_rules! assert_table_api { &$x::clear, &$x::columns, &$x::delete, + &$x::describe, &$x::get_index, &$x::get_limit, &$x::get_client, diff --git a/rust/perspective-client/src/rust/table.rs b/rust/perspective-client/src/rust/table.rs index e7108453a0..e54c5163c2 100644 --- a/rust/perspective-client/src/rust/table.rs +++ b/rust/perspective-client/src/rust/table.rs @@ -205,10 +205,141 @@ pub struct UpdateOptions { #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] pub struct ExprValidationResult { pub expression_schema: Schema, - pub errors: HashMap, + pub errors: HashMap, pub expression_alias: HashMap, } +/// The successful result of [`Table::describe`]: what a [`View`] built from the +/// described config would report, WITHOUT building one. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct Description { + /// The type of every expression, before aggregation. + pub expression_schema: Schema, + + /// Equal to [`View::schema`] of a [`View`] built from the same config. + pub view_schema: Schema, +} + +/// Why [`Table::describe`] rejected a config. +#[derive(Clone, Debug, PartialEq)] +pub enum DescribeError { + /// At least one expression failed to compile. + Expressions { + expression_schema: Schema, + errors: HashMap, + }, + + /// Every expression compiled but the rest of the config is invalid. + Config(String), +} + +impl Display for DescribeError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Expressions { errors, .. } => { + let mut names = errors.keys().collect::>(); + names.sort(); + write!(f, "Invalid expressions: ")?; + for (i, name) in names.iter().enumerate() { + if i > 0 { + write!(f, "; ")?; + } + + write!(f, "{}: {}", name, errors[*name].error_message)?; + } + + Ok(()) + }, + Self::Config(msg) => write!(f, "{}", msg), + } + } +} + +impl std::error::Error for DescribeError {} + +/// The wire/serde shape of a [`Table::describe`] verdict, as exchanged with +/// JavaScript and Python (both client results and virtual-server handler +/// returns): exactly one of the three arms, discriminated by its keys. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(untagged)] +pub enum DescribeVerdict { + Ok { + expression_schema: Schema, + view_schema: Schema, + }, + Expressions { + #[serde(default)] + expression_schema: Schema, + expression_errors: HashMap, + }, + Config { + config_error: String, + }, +} + +impl From> for DescribeVerdict { + fn from(value: Result) -> Self { + match value { + Ok(d) => Self::Ok { + expression_schema: d.expression_schema, + view_schema: d.view_schema, + }, + Err(DescribeError::Expressions { + expression_schema, + errors, + }) => Self::Expressions { + expression_schema, + expression_errors: errors, + }, + Err(DescribeError::Config(config_error)) => Self::Config { config_error }, + } + } +} + +impl TryFrom for Result { + type Error = ClientError; + + /// Fails only for the unrepresentable `Expressions` arm with no errors. + fn try_from(value: DescribeVerdict) -> Result { + Ok(match value { + DescribeVerdict::Ok { + expression_schema, + view_schema, + } => Ok(Description { + expression_schema, + view_schema, + }), + DescribeVerdict::Expressions { + expression_schema, + expression_errors, + } => { + if expression_errors.is_empty() { + return Err(ClientError::Unknown( + "Describe verdict has an empty `expression_errors`".to_string(), + )); + } + + Err(DescribeError::Expressions { + expression_schema, + errors: expression_errors, + }) + }, + DescribeVerdict::Config { config_error } => Err(DescribeError::Config(config_error)), + }) + } +} + +fn decode_schema(schema: HashMap) -> ClientResult { + schema + .into_iter() + .map(|(name, ty)| { + ColumnType::try_from(ty) + .map(|ty| (name, ty)) + .map_err(|e| ClientError::Unknown(e.to_string())) + }) + .collect() +} + /// [`Table`] is Perspective's columnar data frame, analogous to a Pandas/Polars /// `DataFrame` or Apache Arrow, supporting append & in-place updates, removal /// by index, and update notifications. @@ -571,26 +702,72 @@ impl Table { } } + /// Validate a complete [`ViewConfigUpdate`] against this table and report + /// the schema a [`View`] built from it would have, without creating one. + pub async fn describe( + &self, + config: ViewConfigUpdate, + ) -> ClientResult> { + let msg = self.client_message(ClientReq::TableDescribeReq(TableDescribeReq { + config: Some(config.into()), + })); + + let resp = match self.client.oneshot(&msg).await? { + ClientResp::TableDescribeResp(resp) => resp, + resp => return Err(resp.into()), + }; + + let expression_schema = decode_schema(resp.expression_schema)?; + let has_errors = !resp.expression_errors.is_empty(); + Ok(match (resp.result, has_errors) { + (Some(table_describe_resp::Result::View(view)), false) => Ok(Description { + expression_schema, + view_schema: decode_schema(view.schema)?, + }), + (Some(table_describe_resp::Result::ConfigError(msg)), false) => { + Err(DescribeError::Config(msg)) + }, + (None, true) => Err(DescribeError::Expressions { + expression_schema, + errors: resp.expression_errors, + }), + (result, _) => { + return Err(ClientError::Unknown(format!( + "Malformed describe response: result={:?} expression_errors={:?}", + result.is_some(), + resp.expression_errors.keys().collect::>() + ))); + }, + }) + } + /// Validates the given expressions. pub async fn validate_expressions( &self, expressions: Expressions, ) -> ClientResult { - let msg = self.client_message(ClientReq::TableValidateExprReq(TableValidateExprReq { - column_to_expr: expressions.0, - })); + let expression_alias = expressions.0.clone(); + let config = ViewConfigUpdate { + expressions: Some(expressions), + columns: Some(vec![]), + ..ViewConfigUpdate::default() + }; - match self.client.oneshot(&msg).await? { - ClientResp::TableValidateExprResp(result) => Ok(ExprValidationResult { - errors: result.errors, - expression_alias: result.expression_alias, - expression_schema: result - .expression_schema - .into_iter() - .map(|(x, y)| (x, ColumnType::try_from(y).unwrap())) - .collect(), + match self.describe(config).await? { + Ok(d) => Ok(ExprValidationResult { + expression_schema: d.expression_schema, + errors: HashMap::new(), + expression_alias, }), - resp => Err(resp.into()), + Err(DescribeError::Expressions { + expression_schema, + errors, + }) => Ok(ExprValidationResult { + expression_schema, + errors, + expression_alias, + }), + Err(DescribeError::Config(msg)) => Err(ClientError::Internal(msg)), } } diff --git a/rust/perspective-client/src/rust/virtual_server/generic_sql_model.rs b/rust/perspective-client/src/rust/virtual_server/generic_sql_model.rs index e91d8464b0..bee333ba38 100644 --- a/rust/perspective-client/src/rust/virtual_server/generic_sql_model.rs +++ b/rust/perspective-client/src/rust/virtual_server/generic_sql_model.rs @@ -45,7 +45,7 @@ use crate::config::{ FilterTerm, GroupRollupMode, Scalar, Sort, SortDir, SplitRollupMode, ViewConfig, }; use crate::proto::{ColumnType, ViewPort}; -use crate::virtual_server::generic_sql_model::table_make_view::ViewQueryContext; +use crate::virtual_server::generic_sql_model::table_make_view::{ViewQueryContext, quote_ident}; /// Error type for SQL generation operations. #[derive(Debug, Clone)] @@ -117,6 +117,10 @@ pub struct GenericSQLVirtualServerModelArgs { /// ClickHouse (both RE2, matching the engine's semantics). When /// omitted, regex filter clauses are dropped. regex_fn: Option, + + /// Template wrapping a `SELECT` so the dialect PLANS it and reports its + /// result columns without executing it; `{}` is the query. + describe_template: Option, } /// Recovers the source column of a pivoted view column name — the longest @@ -224,23 +228,88 @@ impl GenericSQLVirtualServerModel { Ok(format!("SELECT COUNT(*) FROM (DESCRIBE {})", view_id)) } - /// Returns the SQL query to validate an expression against a table. + fn describe_wrap(&self, query: &str) -> String { + self.0 + .describe_template + .as_deref() + .unwrap_or("DESCRIBE ({})") + .replacen("{}", query, 1) + } + + /// Returns the SQL query that plans every expression of `config` at once + /// against `table_id`. /// - /// # Arguments - /// * `table_id` - The identifier of the table. - /// * `expression` - The SQL expression to validate. + /// # Returns + /// SQL: `DESCRIBE (SELECT {expr} AS "{alias}", ... FROM {table_id})` + pub fn expressions_describe( + &self, + table_id: &str, + config: &ViewConfig, + ) -> GenericSQLResult> { + if config.expressions.is_empty() { + return Ok(None); + } + + let mut names = config.expressions.keys().collect::>(); + names.sort(); + let selects = names + .iter() + .map(|name| { + format!( + "{} AS \"{}\"", + config.expressions.0[*name], + quote_ident(name) + ) + }) + .collect::>() + .join(", "); + + Ok(Some(self.describe_wrap(&format!( + "SELECT {} FROM {}", + selects, table_id + )))) + } + + /// Returns the SQL query that plans one expression against a table. /// /// # Returns /// SQL: `DESCRIBE (SELECT {expression} FROM {table_id})` - pub fn table_validate_expression( + pub fn expression_describe( &self, table_id: &str, expression: &str, ) -> GenericSQLResult { - Ok(format!( - "DESCRIBE (SELECT {} FROM {})", - expression, table_id - )) + Ok(self.describe_wrap(&format!("SELECT {} FROM {}", expression, table_id))) + } + + /// Returns the SQL query that plans the view `config` would build against + /// `table_id`, without materializing it. + /// + /// # Returns + /// SQL: `DESCRIBE (SELECT ... GROUP BY ...)` + pub fn table_describe( + &self, + table_id: &str, + config: &ViewConfig, + schema: &IndexMap, + ) -> GenericSQLResult> { + if config.columns.iter().flatten().next().is_none() { + return Ok(None); + } + + let mut folded = config.clone(); + folded.sort.clear(); + let split_by = std::mem::take(&mut folded.split_by); + if !split_by.is_empty() && folded.group_rollup_mode != GroupRollupMode::Total { + if folded.group_by.is_empty() { + folded.aggregates.clear(); + } + + folded.group_by.extend(split_by); + } + + let ctx = ViewQueryContext::new(self, table_id, &folded, schema)?; + Ok(Some(self.describe_wrap(&ctx.build_query()))) } /// Returns the SQL query to delete a view. diff --git a/rust/perspective-client/src/rust/virtual_server/generic_sql_model/table_make_view.rs b/rust/perspective-client/src/rust/virtual_server/generic_sql_model/table_make_view.rs index 2c0b2b12cc..c5702bbfcb 100644 --- a/rust/perspective-client/src/rust/virtual_server/generic_sql_model/table_make_view.rs +++ b/rust/perspective-client/src/rust/virtual_server/generic_sql_model/table_make_view.rs @@ -262,7 +262,7 @@ fn window_sql( } } -fn quote_ident(name: &str) -> String { +pub(crate) fn quote_ident(name: &str) -> String { name.replace('"', "\"\"") } diff --git a/rust/perspective-client/src/rust/virtual_server/generic_sql_model/tests.rs b/rust/perspective-client/src/rust/virtual_server/generic_sql_model/tests.rs index c98564e3bd..efcdc44ca8 100644 --- a/rust/perspective-client/src/rust/virtual_server/generic_sql_model/tests.rs +++ b/rust/perspective-client/src/rust/virtual_server/generic_sql_model/tests.rs @@ -14,7 +14,8 @@ use std::collections::HashMap; use super::*; use crate::config::{ - Aggregate, GroupRollupMode, WindowFrame, WindowSort, WindowSortDir, WindowSpec, Windows, + Aggregate, Expressions, GroupRollupMode, Sort, SortDir, WindowFrame, WindowSort, WindowSortDir, + WindowSpec, Windows, }; #[test] @@ -1246,3 +1247,140 @@ fn test_table_make_view_grouping_fn_in_rollup_order() { sql ); } + +#[test] +fn test_table_describe_flat() { + let builder = GenericSQLVirtualServerModel::new(GenericSQLVirtualServerModelArgs::default()); + let mut config = ViewConfig::default(); + config.columns = vec![Some("col1".to_string()), Some("col2".to_string())]; + let sql = builder + .table_describe("source_table", &config, &IndexMap::new()) + .unwrap() + .unwrap(); + + assert!( + sql.starts_with( + "DESCRIBE (SELECT \"col1\" as \"col1\", \"col2\" as \"col2\" FROM source_table" + ), + "{sql}" + ); + assert!(sql.ends_with(')'), "{sql}"); +} + +#[test] +fn test_table_describe_no_columns_is_none() { + let builder = GenericSQLVirtualServerModel::new(GenericSQLVirtualServerModelArgs::default()); + let mut config = ViewConfig::default(); + config.expressions = Expressions(HashMap::from([("e".to_string(), "\"a\" + 1".to_string())])); + assert!( + builder + .table_describe("source_table", &config, &IndexMap::new()) + .unwrap() + .is_none() + ); +} + +#[test] +fn test_table_describe_split_by_folds_into_group_by_without_pivot() { + let builder = GenericSQLVirtualServerModel::new(GenericSQLVirtualServerModelArgs::default()); + let mut config = ViewConfig::default(); + config.columns = vec![Some("value".to_string())]; + config.group_by = vec!["state".to_string()]; + config.split_by = vec!["quarter".to_string()]; + config.aggregates = HashMap::from([( + "value".to_string(), + Aggregate::SingleAggregate("sum".to_string()), + )]); + config.sort = vec![Sort("value".to_string(), SortDir::Desc)]; + let sql = builder + .table_describe("t", &config, &IndexMap::new()) + .unwrap() + .unwrap(); + + assert!(!sql.contains("PIVOT"), "{sql}"); + assert!(sql.contains("sum(\"value\") as \"value\""), "{sql}"); + assert!( + sql.contains("GROUP BY ROLLUP(\"state\", \"quarter\")"), + "{sql}" + ); + assert!(!sql.contains("__SORT_"), "{sql}"); +} + +#[test] +fn test_table_describe_column_only_drops_aggregates() { + let builder = GenericSQLVirtualServerModel::new(GenericSQLVirtualServerModelArgs::default()); + let mut config = ViewConfig::default(); + config.columns = vec![Some("value".to_string())]; + config.split_by = vec!["quarter".to_string()]; + config.aggregates = HashMap::from([( + "value".to_string(), + Aggregate::SingleAggregate("avg".to_string()), + )]); + let sql = builder + .table_describe("t", &config, &IndexMap::new()) + .unwrap() + .unwrap(); + + assert!(!sql.contains("PIVOT"), "{sql}"); + assert!(sql.contains("any_value(\"value\") as \"value\""), "{sql}"); + assert!(sql.contains("GROUP BY ROLLUP(\"quarter\")"), "{sql}"); +} + +#[test] +fn test_table_describe_total_split_by_keeps_aggregates() { + let builder = GenericSQLVirtualServerModel::new(GenericSQLVirtualServerModelArgs::default()); + let mut config = ViewConfig::default(); + config.columns = vec![Some("value".to_string())]; + config.split_by = vec!["quarter".to_string()]; + config.group_rollup_mode = GroupRollupMode::Total; + config.aggregates = HashMap::from([( + "value".to_string(), + Aggregate::SingleAggregate("avg".to_string()), + )]); + let sql = builder + .table_describe("t", &config, &IndexMap::new()) + .unwrap() + .unwrap(); + + assert!(!sql.contains("PIVOT"), "{sql}"); + assert!(sql.contains("avg(\"value\") as \"value\""), "{sql}"); + assert!(!sql.contains("GROUP BY"), "{sql}"); + assert!(!sql.contains("__GROUPING_ID__"), "{sql}"); + assert!(!sql.contains("ORDER BY"), "{sql}"); +} + +#[test] +fn test_expressions_describe() { + let builder = GenericSQLVirtualServerModel::new(GenericSQLVirtualServerModelArgs::default()); + let mut config = ViewConfig::default(); + assert!( + builder + .expressions_describe("t", &config) + .unwrap() + .is_none() + ); + config.expressions = Expressions(HashMap::from([ + ("b".to_string(), "\"x\" * 2".to_string()), + ("a".to_string(), "\"x\" + 1".to_string()), + ])); + + assert_eq!( + builder.expressions_describe("t", &config).unwrap().unwrap(), + "DESCRIBE (SELECT \"x\" + 1 AS \"a\", \"x\" * 2 AS \"b\" FROM t)" + ); +} + +#[test] +fn test_expression_describe_uses_template() { + let builder = GenericSQLVirtualServerModel::new( + serde_json::from_value(serde_json::json!({ + "describe_template": "SELECT * FROM ({}) AS __psp_describe__ LIMIT 0" + })) + .unwrap(), + ); + + assert_eq!( + builder.expression_describe("t", "\"x\" + 1").unwrap(), + "SELECT * FROM (SELECT \"x\" + 1 FROM t) AS __psp_describe__ LIMIT 0" + ); +} diff --git a/rust/perspective-client/src/rust/virtual_server/handler.rs b/rust/perspective-client/src/rust/virtual_server/handler.rs index 6b7f013500..edf286b914 100644 --- a/rust/perspective-client/src/rust/virtual_server/handler.rs +++ b/rust/perspective-client/src/rust/virtual_server/handler.rs @@ -17,8 +17,10 @@ use indexmap::IndexMap; use super::data::VirtualDataSlice; use super::features::Features; +use super::generic_sql_model::column_path_source; use crate::config::{ViewConfig, ViewConfigUpdate}; use crate::proto::{ColumnType, HostedTable, TableMakePortReq, ViewPort}; +use crate::table::{DescribeError, Description}; #[cfg(feature = "sendable")] pub type VirtualServerFuture<'a, T> = Pin + 'a>>; @@ -70,6 +72,14 @@ pub trait VirtualServerHandler { config: &mut ViewConfigUpdate, ) -> VirtualServerFuture<'_, Result>; + /// Validates a complete view config against a table and reports the schema + /// a view built from it would have, WITHOUT creating one. + fn table_describe( + &mut self, + table_id: &str, + config: &ViewConfig, + ) -> VirtualServerFuture<'_, Result, Self::Error>>; + /// Deletes a view and releases its resources. fn view_delete(&self, view_id: &str) -> VirtualServerFuture<'_, Result<(), Self::Error>>; @@ -117,17 +127,6 @@ pub trait VirtualServerHandler { Box::pin(self.table_schema(view_id)) } - /// Validates an expression against a table and returns its result type. - /// - /// Default implementation returns `Float` for all expressions. - fn table_validate_expression( - &self, - _table_id: &str, - _expression: &str, - ) -> VirtualServerFuture<'_, Result> { - Box::pin(async { Ok(ColumnType::Float) }) - } - /// Returns the features supported by this handler. /// /// Default implementation returns default features. @@ -171,3 +170,54 @@ pub trait VirtualServerHandler { Box::pin(async { unimplemented!("make_table not implemented") }) } } + +/// A [`VirtualServerHandler::table_describe`] implementation for backends with +/// no cheaper native answer: build a real view on a private id, read its +/// schema, delete it. +pub async fn describe_via_make_view( + handler: &mut H, + table_id: &str, + config: &ViewConfig, +) -> Result, H::Error> { + static COUNTER: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + let view_id = format!( + "__psp_describe_{}__", + COUNTER.fetch_add(1, std::sync::atomic::Ordering::Relaxed) + ); + + let mut update: ViewConfigUpdate = config.clone().into(); + let view_id = match handler + .table_make_view(table_id, &view_id, &mut update) + .await + { + Ok(view_id) => view_id, + Err(e) => return Ok(Err(DescribeError::Config(e.to_string()))), + }; + + let config: ViewConfig = update.into(); + let schema = handler.view_schema(&view_id, &config).await; + let deleted = handler.view_delete(&view_id).await; + let schema = schema?; + deleted?; + let view_schema = schema + .iter() + .map(|(name, ty)| { + let source = column_path_source(name, &config) + .map(|(_, col)| col.to_string()) + .unwrap_or_else(|| name.clone()); + + (source, *ty) + }) + .collect::>(); + + let expression_schema = config + .expressions + .keys() + .filter_map(|name| view_schema.get(name).map(|ty| (name.clone(), *ty))) + .collect(); + + Ok(Ok(Description { + expression_schema, + view_schema, + })) +} diff --git a/rust/perspective-client/src/rust/virtual_server/mod.rs b/rust/perspective-client/src/rust/virtual_server/mod.rs index 88208d2cf6..f89ecee423 100644 --- a/rust/perspective-client/src/rust/virtual_server/mod.rs +++ b/rust/perspective-client/src/rust/virtual_server/mod.rs @@ -29,5 +29,5 @@ pub use generic_sql_model::{ GenericSQLError, GenericSQLResult, GenericSQLVirtualServerModel, GenericSQLVirtualServerModelArgs, }; -pub use handler::{VirtualServerFuture, VirtualServerHandler}; +pub use handler::{VirtualServerFuture, VirtualServerHandler, describe_via_make_view}; pub use server::VirtualServer; diff --git a/rust/perspective-client/src/rust/virtual_server/server.rs b/rust/perspective-client/src/rust/virtual_server/server.rs index 3b4642808d..43d898ed36 100644 --- a/rust/perspective-client/src/rust/virtual_server/server.rs +++ b/rust/perspective-client/src/rust/virtual_server/server.rs @@ -22,16 +22,17 @@ use super::generic_sql_model::{column_path_source, sort_column_paths}; use super::handler::VirtualServerHandler; use crate::config::{ViewConfig, ViewConfigUpdate}; use crate::proto::response::ClientResp; -use crate::proto::table_validate_expr_resp::ExprValidationError; use crate::proto::{ ColumnType, GetFeaturesResp, GetHostedTablesResp, MakeTableResp, Request, Response, - ServerError, TableMakePortResp, TableMakeViewResp, TableOnDeleteResp, TableRemoveDeleteResp, - TableSchemaResp, TableSizeResp, TableValidateExprResp, ViewColumnPathsResp, ViewDeleteResp, - ViewDimensionsResp, ViewExpressionSchemaResp, ViewGetConfigResp, ViewGetMinMaxResp, - ViewOnDeleteResp, ViewOnRemoveResp, ViewOnUpdateResp, ViewRemoveDeleteResp, + ServerError, TableDescribeResp, TableMakePortResp, TableMakeViewResp, TableOnDeleteResp, + TableRemoveDeleteResp, TableSchemaResp, TableSizeResp, ViewColumnPathsResp, ViewDeleteResp, + ViewDescription, ViewDimensionsResp, ViewExpressionSchemaResp, ViewGetConfigResp, + ViewGetMinMaxResp, ViewOnDeleteResp, ViewOnRemoveResp, ViewOnUpdateResp, ViewRemoveDeleteResp, ViewRemoveOnRemoveResp, ViewRemoveOnUpdateResp, ViewSchemaResp, ViewToArrowResp, ViewToColumnsStringResp, ViewToCsvResp, ViewToNdjsonStringResp, ViewToRowsStringResp, + table_describe_resp, }; +use crate::table::{DescribeError, Description}; macro_rules! respond { ($msg:ident, $name:ident { $($rest:tt)* }) => {{ @@ -60,6 +61,10 @@ pub struct VirtualServer { view_to_table: IndexMap, view_configs: IndexMap, view_schemas: IndexMap>, + + /// Per-view `table_describe` answers, computed LAZILY on the first + /// `ViewExpressionSchemaReq` for that view — never on view creation. + view_descriptions: IndexMap, } impl VirtualServer { @@ -70,6 +75,7 @@ impl VirtualServer { view_configs: IndexMap::default(), view_to_table: IndexMap::default(), view_schemas: IndexMap::default(), + view_descriptions: IndexMap::default(), } } @@ -206,34 +212,43 @@ impl VirtualServer { size: self.handler.table_size(msg.entity_id.as_str()).await? }) }, - TableValidateExprReq(req) => { - let mut expression_schema = HashMap::::default(); - let mut expression_alias = HashMap::::default(); - let mut errors = HashMap::::default(); - for (name, ex) in req.column_to_expr.iter() { - let _ = expression_alias.insert(name.clone(), ex.clone()); - match self - .handler - .table_validate_expression(&msg.entity_id, ex.as_str()) - .await - { - Ok(dtype) => { - let _ = expression_schema.insert(name.clone(), dtype as i32); - }, - Err(e) => { - let _ = errors.insert(name.clone(), ExprValidationError { - error_message: format!("{}", e), - line: 0, - column: 0, - }); - }, - } - } + TableDescribeReq(req) => { + let config: ViewConfig = req.config.unwrap_or_default().into(); + let verdict = self + .handler + .table_describe(msg.entity_id.as_str(), &config) + .await?; + + let (expression_schema, expression_errors, result) = match verdict { + Ok(d) => ( + d.expression_schema, + HashMap::new(), + Some(table_describe_resp::Result::View(ViewDescription { + schema: d + .view_schema + .into_iter() + .map(|(x, y)| (x, y as i32)) + .collect(), + })), + ), + Err(DescribeError::Expressions { + expression_schema, + errors, + }) => (expression_schema, errors, None), + Err(DescribeError::Config(msg)) => ( + HashMap::new(), + HashMap::new(), + Some(table_describe_resp::Result::ConfigError(msg)), + ), + }; - respond!(msg, TableValidateExprResp { - expression_schema, - errors, - expression_alias, + respond!(msg, TableDescribeResp { + expression_schema: expression_schema + .into_iter() + .map(|(x, y)| (x, y as i32)) + .collect(), + expression_errors, + result, }) }, ViewSchemaReq(_) => { @@ -278,31 +293,31 @@ impl VirtualServer { }) }, ViewExpressionSchemaReq(_) => { - let mut schema = HashMap::::default(); - let table_id = self.view_to_table.get(&msg.entity_id); - for (name, ex) in self - .view_configs - .get(&msg.entity_id) - .unwrap() - .expressions - .iter() - { - match self + let view_id = msg.entity_id.clone(); + if !self.view_descriptions.contains_key(&view_id) { + let table_id = self + .view_to_table + .get(&view_id) + .cloned() + .ok_or_else(|| VirtualServerError::UnknownViewId(view_id.clone()))?; + + let config = self.view_configs.get(&view_id).unwrap().clone(); + let description = self .handler - .table_validate_expression(table_id.unwrap(), ex.as_str()) - .await - { - Ok(dtype) => { - let _ = schema.insert(name.clone(), dtype as i32); - }, - Err(_e) => { - // TODO: handle error - }, - } + .table_describe(&table_id, &config) + .await? + .map_err(|e| VirtualServerError::Other(e.to_string()))?; + + self.view_descriptions.insert(view_id.clone(), description); } - let resp = ViewExpressionSchemaResp { schema }; - respond!(msg, ViewExpressionSchemaResp { ..resp }) + let schema = self.view_descriptions[&view_id] + .expression_schema + .iter() + .map(|(name, ty)| (name.clone(), *ty as i32)) + .collect(); + + respond!(msg, ViewExpressionSchemaResp { schema }) }, ViewColumnPathsReq(view_column_paths_req) => { let config = self.view_configs.get(&msg.entity_id).unwrap(); @@ -426,6 +441,7 @@ impl VirtualServer { self.handler.view_delete(msg.entity_id.as_str()).await?; self.view_to_table.shift_remove(&msg.entity_id); self.view_configs.shift_remove(&msg.entity_id); + self.view_descriptions.shift_remove(&msg.entity_id); respond!(msg, ViewDeleteResp {}) }, MakeTableReq(req) => { diff --git a/rust/perspective-js/src/rust/generic_sql_model.rs b/rust/perspective-js/src/rust/generic_sql_model.rs index 41d739d5f2..ce8ce57dd2 100644 --- a/rust/perspective-js/src/rust/generic_sql_model.rs +++ b/rust/perspective-js/src/rust/generic_sql_model.rs @@ -83,15 +83,54 @@ impl GenericSQLVirtualServerModel { .map_err(|e| JsValue::from_str(&e.to_string())) } - /// Returns the SQL query to validate an expression against a table. - #[wasm_bindgen(js_name = "tableValidateExpression")] - pub fn table_validate_expression( + /// Returns the SQL query that plans every expression of `config` at once + /// against a table, one result column per expression alias, or `undefined` + /// when the config has no expressions. + #[wasm_bindgen(js_name = "expressionsDescribe")] + pub fn expressions_describe( &self, table_id: &str, - expression: &str, - ) -> Result { + config: JsValue, + ) -> Result, JsValue> { + let config: ViewConfig = serde_wasm_bindgen::from_value(config) + .map_err(|e| JsValue::from_str(&e.to_string()))?; + + self.inner + .expressions_describe(table_id, &config) + .map_err(|e| JsValue::from_str(&e.to_string())) + } + + /// Returns the SQL query that plans ONE expression against a table - for + /// attributing a failed `expressionsDescribe` to the expressions at fault. + #[wasm_bindgen(js_name = "expressionDescribe")] + pub fn expression_describe(&self, table_id: &str, expression: &str) -> Result { + self.inner + .expression_describe(table_id, expression) + .map_err(|e| JsValue::from_str(&e.to_string())) + } + + /// Returns the SQL query that plans the view `config` would build and + /// reports its result columns without materializing it (`split_by` folded + /// into `group_by`, no `PIVOT`), or `undefined` when the config selects no + /// columns. + #[wasm_bindgen(js_name = "tableDescribe")] + pub fn table_describe( + &self, + table_id: &str, + config: JsValue, + schema: JsValue, + ) -> Result, JsValue> { + let config: ViewConfig = serde_wasm_bindgen::from_value(config) + .map_err(|e| JsValue::from_str(&e.to_string()))?; + + let schema = if schema.is_undefined() || schema.is_null() { + IndexMap::new() + } else { + self.parse_schema(schema)? + }; + self.inner - .table_validate_expression(table_id, expression) + .table_describe(table_id, &config, &schema) .map_err(|e| JsValue::from_str(&e.to_string())) } diff --git a/rust/perspective-js/src/rust/table.rs b/rust/perspective-js/src/rust/table.rs index 20665791f5..f0dc8269dd 100644 --- a/rust/perspective-js/src/rust/table.rs +++ b/rust/perspective-js/src/rust/table.rs @@ -12,7 +12,9 @@ use js_sys::Function; use perspective_client::config::*; -use perspective_client::{DeleteOptions, UpdateData, UpdateOptions, assert_table_api}; +use perspective_client::{ + DeleteOptions, DescribeVerdict, UpdateData, UpdateOptions, assert_table_api, +}; use wasm_bindgen::prelude::*; use wasm_bindgen_derive::TryFromJsValue; use wasm_bindgen_futures::spawn_local; @@ -355,6 +357,36 @@ impl Table { Ok(View(view)) } + /// Validate a complete view config against this table and report the + /// schema a [`View`] built from it would have, WITHOUT creating one. + /// + /// # JavaScript Examples + /// + /// ```javascript + /// const verdict = await table.describe({ + /// columns: ["Sales", "margin"], + /// group_by: ["Region"], + /// expressions: { margin: '"Profit" / "Sales"' }, + /// }); + /// + /// if ("view_schema" in verdict) { + /// const view = await table.view(config); + /// } + /// ``` + #[wasm_bindgen] + pub async fn describe(&self, config: Option) -> ApiResult { + let config = config + .map(|config| js_sys::JSON::stringify(&config)) + .transpose()? + .and_then(|x| x.as_string()) + .map(|x| serde_json::from_str(x.as_str())) + .transpose()? + .unwrap_or_default(); + + let verdict: DescribeVerdict = self.0.describe(config).await?.into(); + Ok(JsValue::from_serde_ext(&verdict)?) + } + /// Validates the given expressions. #[wasm_bindgen] pub async fn validate_expressions(&self, exprs: &JsValue) -> ApiResult { diff --git a/rust/perspective-js/src/rust/virtual_server.rs b/rust/perspective-js/src/rust/virtual_server.rs index 32c7524e4a..8aeb60acfb 100644 --- a/rust/perspective-js/src/rust/virtual_server.rs +++ b/rust/perspective-js/src/rust/virtual_server.rs @@ -20,8 +20,8 @@ use std::sync::{Arc, Mutex}; use indexmap::IndexMap; use js_sys::{Array, Date, Object, Reflect, Uint8Array}; use perspective_client::proto::{ColumnType, HostedTable}; -use perspective_client::virtual_server; use perspective_client::virtual_server::{Features, ResultExt, VirtualServerHandler}; +use perspective_client::{DescribeError, DescribeVerdict, Description, virtual_server}; use serde::Serialize; use wasm_bindgen::prelude::*; use wasm_bindgen_futures::JsFuture; @@ -91,6 +91,25 @@ export interface VirtualHostedTable { limit?: number; } +/** One expression's compile error, as reported by `tableDescribe`. */ +export interface ExpressionError { + error_message: string; + line: number; + column: number; +} + +/** The verdict of `VirtualServerHandler.tableDescribe`. */ +export type TableDescription = + | { + expression_schema: Record; + view_schema: Record; + } + | { + expression_schema?: Record; + expression_errors: Record; + } + | { config_error: string }; + /** * Handler interface that you implement to provide custom data sources. * @@ -139,11 +158,14 @@ export interface VirtualServerHandler { config: ViewConfig, ): number | Promise; - /** Required when `getFeatures()` reports `expressions: true`. */ - tableValidateExpression?( + /** + * Validate a complete view config against a table and report the schema + * a view built from it would have, without creating one. + */ + tableDescribe( tableId: string, - expression: string, - ): ColumnType | Promise; + config: ViewConfig, + ): TableDescription | Promise; viewGetMinMax?( viewId: string, @@ -323,39 +345,39 @@ impl VirtualServerHandler for JsServerHandler { }) } - fn table_validate_expression( - &self, + fn table_describe( + &mut self, table_id: &str, - expression: &str, - ) -> HandlerFuture> { - // TODO Cache these inspection calls - let has_method = Reflect::get(&self.0, &JsValue::from_str("tableValidateExpression")) + config: &perspective_client::config::ViewConfig, + ) -> HandlerFuture, Self::Error>> { + let has_method = Reflect::get(&self.0, &JsValue::from_str("tableDescribe")) .map(|val| !val.is_undefined()) .unwrap_or(false); let handler = self.0.clone(); let table_id = table_id.to_string(); - let expression = expression.to_string(); + let config_value = JsValue::from_serde_ext(config); Box::pin(async move { if !has_method { return Err(JsError(JsValue::from_str( - "feature `table_validate_expression` not implemented", + "`tableDescribe` is required of a `VirtualServerHandler`", ))); } let this = JsServerHandler(handler); let args = Array::new(); args.push(&JsValue::from_str(&table_id)); - args.push(&JsValue::from_str(&expression)); - let result = this - .call_method_js_async("tableValidateExpression", &args) - .await?; - - let type_str = result - .as_string() - .ok_or_else(|| JsError(JsValue::from_str("Must return a string")))?; + args.push(&config_value?); + let result = this.call_method_js_async("tableDescribe", &args).await?; + let verdict: DescribeVerdict = result.into_serde_ext().map_err(|e| { + JsError(JsValue::from_str(&format!( + "`tableDescribe` must return a `TableDescription`: {}", + e + ))) + })?; - Ok(ColumnType::from_str(&type_str).unwrap()) + Result::::try_from(verdict) + .map_err(|e| JsError(JsValue::from_str(&e.to_string()))) }) } diff --git a/rust/perspective-js/src/ts/virtual_servers/clickhouse.ts b/rust/perspective-js/src/ts/virtual_servers/clickhouse.ts index 46395902d8..866ed92b86 100644 --- a/rust/perspective-js/src/ts/virtual_servers/clickhouse.ts +++ b/rust/perspective-js/src/ts/virtual_servers/clickhouse.ts @@ -24,6 +24,10 @@ import type * as perspective from "@perspective-dev/client"; import type { ColumnType } from "@perspective-dev/client/dist/esm/ts-rs/ColumnType.d.ts"; import type { ViewConfig } from "@perspective-dev/client/dist/esm/ts-rs/ViewConfig.d.ts"; +import type { + ExpressionError, + TableDescription, +} from "@perspective-dev/client"; import type { ViewConfigUpdate } from "@perspective-dev/client/dist/esm/ts-rs/ViewConfigUpdate.d.ts"; import type { ViewWindow } from "@perspective-dev/client/dist/esm/ts-rs/ViewWindow.d.ts"; import type { WindowAggSpec } from "@perspective-dev/client/dist/esm/ts-rs/WindowAggSpec.d.ts"; @@ -374,13 +378,98 @@ export class ClickhouseHandler implements perspective.VirtualServerHandler { await runQuery(this.db, query, { execute: true }); } - async tableValidateExpression(tableId: string, expression: string) { - const query = this.sqlBuilder.tableValidateExpression( + async tableDescribe( + tableId: string, + config: ViewConfig, + ): Promise { + let expression_schema = {} as Record; + const expressions_query = this.sqlBuilder.expressionsDescribe( + tableId, + config, + ); + + if (expressions_query !== undefined) { + try { + expression_schema = await this.describeQuery(expressions_query); + } catch (error) { + return await this.attributeExpressionErrors( + tableId, + config, + error, + ); + } + } + + const schema = Object.keys(config.windows ?? {}).length + ? await this.tableSchema(tableId) + : undefined; + + const view_query = this.sqlBuilder.tableDescribe( tableId, - expression, + config, + schema, ); + + let view_schema = {} as Record; + if (view_query !== undefined) { + try { + view_schema = await this.describeQuery(view_query); + } catch (error) { + return { config_error: errorMessage(error) }; + } + } + + return { expression_schema, view_schema }; + } + + /** The planned result columns of one `DESCRIBE` query. */ + private async describeQuery(query: string) { const results = await runQuery(this.db, query); - return duckdbTypeToPsp(results[0]["type"]) as ColumnType; + const schema = {} as Record; + for (const result of results) { + if (!result.name.startsWith("__")) { + schema[result.name] = duckdbTypeToPsp( + result.type, + ) as ColumnType; + } + } + + return schema; + } + + /** Plans each expression on its own to name the ones at fault. */ + private async attributeExpressionErrors( + tableId: string, + config: ViewConfig, + error: unknown, + ): Promise { + const expression_schema = {} as Record; + const expression_errors = {} as Record; + for (const [name, expression] of Object.entries(config.expressions)) { + const query = this.sqlBuilder.expressionDescribe( + tableId, + expression, + ); + + try { + const results = await runQuery(this.db, query); + expression_schema[name] = duckdbTypeToPsp( + results[0]["type"], + ) as ColumnType; + } catch (error) { + expression_errors[name] = { + error_message: errorMessage(error), + line: 0, + column: 0, + }; + } + } + + if (Object.keys(expression_errors).length === 0) { + return { config_error: errorMessage(error) }; + } + + return { expression_schema, expression_errors }; } async viewDelete(viewId: string) { @@ -436,3 +525,7 @@ export class ClickhouseHandler implements perspective.VirtualServerHandler { } } } + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} diff --git a/rust/perspective-js/src/ts/virtual_servers/duckdb.ts b/rust/perspective-js/src/ts/virtual_servers/duckdb.ts index 3eef948a8c..68a5bbe3fa 100644 --- a/rust/perspective-js/src/ts/virtual_servers/duckdb.ts +++ b/rust/perspective-js/src/ts/virtual_servers/duckdb.ts @@ -24,6 +24,10 @@ import type * as perspective from "@perspective-dev/client"; import type { ColumnType } from "@perspective-dev/client/dist/esm/ts-rs/ColumnType.d.ts"; import type { ViewConfig } from "@perspective-dev/client/dist/esm/ts-rs/ViewConfig.d.ts"; +import type { + ExpressionError, + TableDescription, +} from "@perspective-dev/client"; import type { ViewConfigUpdate } from "@perspective-dev/client/dist/esm/ts-rs/ViewConfigUpdate.d.ts"; import type { ViewWindow } from "@perspective-dev/client/dist/esm/ts-rs/ViewWindow.d.ts"; import type { WindowAggSpec } from "@perspective-dev/client/dist/esm/ts-rs/WindowAggSpec.d.ts"; @@ -386,15 +390,99 @@ export class DuckDBHandler implements perspective.VirtualServerHandler { await runQuery(this.db, query); } - async tableValidateExpression(tableId: string, expression: string) { - const query = this.sqlBuilder.tableValidateExpression( + async tableDescribe( + tableId: string, + config: ViewConfig, + ): Promise { + let expression_schema = {} as Record; + const expressions_query = this.sqlBuilder.expressionsDescribe( tableId, - expression, + config, ); + + if (expressions_query !== undefined) { + try { + expression_schema = await this.describeQuery(expressions_query); + } catch (error) { + return await this.attributeExpressionErrors( + tableId, + config, + error, + ); + } + } + + const schema = Object.keys(config.windows ?? {}).length + ? await this.tableSchema(tableId) + : undefined; + + const view_query = this.sqlBuilder.tableDescribe( + tableId, + config, + schema, + ); + + let view_schema = {} as Record; + if (view_query !== undefined) { + try { + view_schema = await this.describeQuery(view_query); + } catch (error) { + return { config_error: errorMessage(error) }; + } + } + + return { expression_schema, view_schema }; + } + + /** The planned result columns of one `DESCRIBE` query. */ + private async describeQuery(query: string) { const results = await runQuery(this.db, query); - return duckdbTypeToPsp( - results[0].toJSON()["column_type"], - ) as ColumnType; + const schema = {} as Record; + for (const result of results) { + const res = result.toJSON(); + if (!res.column_name.startsWith("__")) { + schema[res.column_name] = duckdbTypeToPsp( + res.column_type, + ) as ColumnType; + } + } + + return schema; + } + + /** Plans each expression on its own to name the ones at fault. */ + private async attributeExpressionErrors( + tableId: string, + config: ViewConfig, + error: unknown, + ): Promise { + const expression_schema = {} as Record; + const expression_errors = {} as Record; + for (const [name, expression] of Object.entries(config.expressions)) { + const query = this.sqlBuilder.expressionDescribe( + tableId, + expression!, + ); + + try { + const results = await runQuery(this.db, query); + expression_schema[name] = duckdbTypeToPsp( + results[0].toJSON()["column_type"], + ) as ColumnType; + } catch (error) { + expression_errors[name] = { + error_message: errorMessage(error), + line: 0, + column: 0, + }; + } + } + + if (Object.keys(expression_errors).length === 0) { + return { config_error: errorMessage(error) }; + } + + return { expression_schema, expression_errors }; } async viewDelete(viewId: string) { @@ -437,3 +525,7 @@ export class DuckDBHandler implements perspective.VirtualServerHandler { dataSlice.fromArrowIpc(ipc); } } + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} diff --git a/rust/perspective-js/test/js/describe.spec.js b/rust/perspective-js/test/js/describe.spec.js new file mode 100644 index 0000000000..bfc7ade125 --- /dev/null +++ b/rust/perspective-js/test/js/describe.spec.js @@ -0,0 +1,248 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +import { test, expect } from "@perspective-dev/test"; +import perspective from "./perspective_client"; + +async function mixed_table(perspective) { + const table = await perspective.table({ + a: "integer", + b: "float", + c: "string", + d: "boolean", + e: "date", + f: "datetime", + }); + + await table.update({ + a: [1, 2, 3, 4], + b: [1.0, 2.5, 3.0, 4.5], + c: ["x", "y", "z", "w"], + d: [true, false, true, false], + e: [ + new Date(2020, 0, 1), + new Date(2020, 0, 2), + new Date(2020, 0, 3), + new Date(2020, 0, 4), + ], + f: [ + new Date(2020, 0, 1, 1), + new Date(2020, 0, 2, 1), + new Date(2020, 0, 3, 1), + new Date(2020, 0, 4, 1), + ], + }); + + return table; +} + +const VALID_CONFIGS = { + flat: { columns: ["a", "b", "c", "d", "e", "f"] }, + group_by: { columns: ["a", "b", "c"], group_by: ["c"] }, + "group_by with typed aggregates": { + columns: ["a", "b", "c", "e"], + group_by: ["c"], + aggregates: { a: "count", b: "avg", c: "distinct count", e: "last" }, + }, + "group_by with hidden sort": { + columns: ["a"], + group_by: ["c"], + sort: [["b", "desc"]], + }, + split_by: { columns: ["a", "b", "c"], split_by: ["d"] }, + "split_by keeps source types": { + columns: ["a", "b"], + split_by: ["d"], + aggregates: { a: "avg", b: "count" }, + }, + "group_by and split_by": { + columns: ["a", "b"], + group_by: ["c"], + split_by: ["d"], + aggregates: { a: "avg" }, + }, + "split_rollup_mode rollup": { + columns: ["a"], + group_by: ["c"], + split_by: ["d"], + split_rollup_mode: "rollup", + }, + "group_rollup_mode total": { + columns: ["a", "b", "c"], + group_rollup_mode: "total", + }, + "group_rollup_mode total with split_by": { + columns: ["a", "c"], + split_by: ["d"], + group_rollup_mode: "total", + aggregates: { a: "avg" }, + }, + "group_rollup_mode flat": { + columns: ["a", "b"], + group_by: ["c"], + group_rollup_mode: "flat", + }, + expressions: { + columns: ["a", "x", "y"], + expressions: { x: '"a" * 2', y: "'abc'" }, + }, + "expressions aggregated": { + columns: ["x", "y"], + group_by: ["c"], + expressions: { x: '"a" * 2', y: "'abc'" }, + aggregates: { x: "count" }, + }, + windows: { + columns: ["a", "w"], + windows: { + w: { column: "a", aggregate: "sum", order_by: ["e", "asc"] }, + }, + }, + "windows aggregated": { + columns: ["w"], + group_by: ["c"], + windows: { + w: { column: "a", aggregate: "avg", order_by: ["e", "asc"] }, + }, + }, + "no columns": { columns: [], expressions: { x: '"a" + 1' } }, +}; + +((perspective) => { + test.describe("Table.describe()", function () { + test.describe("parity with Table.view()", function () { + for (const [name, config] of Object.entries(VALID_CONFIGS)) { + test(name, async function () { + const table = await mixed_table(perspective); + const verdict = await table.describe(config); + const view = await table.view(config); + const validated = await table.validate_expressions( + config.expressions ?? {}, + ); + + expect(verdict).toEqual({ + expression_schema: validated.expression_schema, + view_schema: await view.schema(), + }); + + await view.delete(); + await table.delete(); + }); + } + }); + + test("reports every invalid expression and the types of the valid ones", async function () { + const table = await mixed_table(perspective); + const config = { + columns: ["a", "good"], + expressions: { + good: '"a" + 1', + bad_column: '"nope" + 1', + bad_syntax: "for () {}", + }, + }; + + const verdict = await table.describe(config); + expect(Object.keys(verdict).sort()).toEqual([ + "expression_errors", + "expression_schema", + ]); + + expect(verdict.expression_schema).toEqual({ good: "float" }); + expect(Object.keys(verdict.expression_errors).sort()).toEqual([ + "bad_column", + "bad_syntax", + ]); + + for (const err of Object.values(verdict.expression_errors)) { + expect(typeof err.error_message).toBe("string"); + expect(err.error_message.length).toBeGreaterThan(0); + expect(typeof err.line).toBe("number"); + expect(typeof err.column).toBe("number"); + } + + await expect(table.view(config)).rejects.toThrow(); + await table.delete(); + }); + + test("reports an invalid column as a config error", async function () { + const table = await mixed_table(perspective); + const config = { columns: ["a", "nope"] }; + const verdict = await table.describe(config); + expect(Object.keys(verdict)).toEqual(["config_error"]); + expect(verdict.config_error).toContain("nope"); + await expect(table.view(config)).rejects.toThrow(); + await table.delete(); + }); + + test("reports an invalid window as a config error", async function () { + const table = await mixed_table(perspective); + const config = { + columns: ["a", "w"], + windows: { + w: { + column: "c", + aggregate: "sum", + order_by: ["e", "asc"], + }, + }, + }; + + const verdict = await table.describe(config); + expect(Object.keys(verdict)).toEqual(["config_error"]); + await expect(table.view(config)).rejects.toThrow(); + await table.delete(); + }); + + test("an expression error is reported before a config error", async function () { + const table = await mixed_table(perspective); + const verdict = await table.describe({ + columns: ["nope"], + expressions: { bad: '"nope" + 1' }, + }); + + expect(Object.keys(verdict).sort()).toEqual([ + "expression_errors", + "expression_schema", + ]); + + await table.delete(); + }); + + test("creates no View", async function () { + const table = await mixed_table(perspective); + for (const config of Object.values(VALID_CONFIGS)) { + await table.describe(config); + } + + await table.delete(); + }); + + test("validate_expressions is a specialization of describe", async function () { + const table = await mixed_table(perspective); + const expressions = { + good: '"a" + 1', + also_good: "'str'", + bad: '"nope" + 1', + }; + + const validated = await table.validate_expressions(expressions); + const verdict = await table.describe({ columns: [], expressions }); + expect(validated.expression_schema).toEqual( + verdict.expression_schema, + ); + expect(validated.errors).toEqual(verdict.expression_errors); + expect(validated.expression_alias).toEqual(expressions); + await table.delete(); + }); + }); +})(perspective); diff --git a/rust/perspective-js/test/js/duckdb/describe.spec.js b/rust/perspective-js/test/js/duckdb/describe.spec.js new file mode 100644 index 0000000000..bc51e52209 --- /dev/null +++ b/rust/perspective-js/test/js/duckdb/describe.spec.js @@ -0,0 +1,159 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +import { test, expect } from "@perspective-dev/test"; +import { describeDuckDB } from "./setup.js"; + +const VALID_CONFIGS = { + flat: { columns: ["Sales", "Quantity", "Region"] }, + group_by: { + columns: ["Sales", "Quantity"], + group_by: ["Region"], + aggregates: { Sales: "sum", Quantity: "avg" }, + }, + "group_by flat": { + columns: ["Sales", "Quantity"], + group_by: ["Region"], + group_rollup_mode: "flat", + }, + "group_by with hidden sort": { + columns: ["Sales"], + group_by: ["Region"], + sort: [["Quantity", "desc"]], + }, + split_by: { columns: ["Sales", "Quantity"], split_by: ["Region"] }, + "split_by keeps source types": { + columns: ["Sales", "Quantity"], + split_by: ["Region"], + aggregates: { Quantity: "avg" }, + }, + "group_by and split_by": { + columns: ["Sales", "Quantity"], + group_by: ["Category"], + split_by: ["Region"], + aggregates: { Sales: "sum", Quantity: "avg" }, + }, + "group_by and split_by flat": { + columns: ["Sales"], + group_by: ["Category"], + split_by: ["Region"], + group_rollup_mode: "flat", + }, + "split_rollup_mode rollup": { + columns: ["Sales"], + group_by: ["Category"], + split_by: ["Region"], + split_rollup_mode: "rollup", + }, + "group_rollup_mode total": { + columns: ["Sales", "Quantity"], + group_rollup_mode: "total", + aggregates: { Quantity: "avg" }, + }, + "group_rollup_mode total with split_by": { + columns: ["Sales", "Quantity"], + split_by: ["Region"], + group_rollup_mode: "total", + aggregates: { Quantity: "avg" }, + }, + expressions: { + columns: ["Sales", "double"], + expressions: { double: '"Sales" * 2' }, + }, + "expressions aggregated": { + columns: ["total"], + group_by: ["Region"], + expressions: { total: '"Sales" + "Profit"' }, + aggregates: { total: "sum" }, + }, +}; + +describeDuckDB("describe", (getClient) => { + for (const [name, config] of Object.entries(VALID_CONFIGS)) { + test(`parity with view: ${name}`, async function () { + const table = await getClient().open_table("memory.superstore"); + const verdict = await table.describe(config); + const view = await table.view(config); + expect(Object.keys(verdict).sort()).toEqual([ + "expression_schema", + "view_schema", + ]); + + expect(verdict.view_schema).toEqual(await view.schema()); + expect(Object.keys(verdict.expression_schema).sort()).toEqual( + Object.keys(config.expressions ?? {}).sort(), + ); + + await view.delete(); + }); + } + + test("attributes a bad expression to its name", async function () { + const table = await getClient().open_table("memory.superstore"); + const config = { + columns: ["Sales", "good", "bad"], + expressions: { + good: '"Sales" * 2', + bad: '"nope" * 2', + }, + }; + + const verdict = await table.describe(config); + expect(Object.keys(verdict).sort()).toEqual([ + "expression_errors", + "expression_schema", + ]); + + expect(verdict.expression_schema).toEqual({ good: "float" }); + expect(Object.keys(verdict.expression_errors)).toEqual(["bad"]); + expect(verdict.expression_errors.bad.error_message).toContain("nope"); + await expect(table.view(config)).rejects.toThrow(); + }); + + test("reports a bad column as a config error", async function () { + const table = await getClient().open_table("memory.superstore"); + const config = { columns: ["Sales", "nope"] }; + const verdict = await table.describe(config); + expect(Object.keys(verdict)).toEqual(["config_error"]); + await expect(table.view(config)).rejects.toThrow(); + }); + + test("an unused invalid expression is still rejected", async function () { + const table = await getClient().open_table("memory.superstore"); + const verdict = await table.describe({ + columns: ["Sales"], + expressions: { bad: '"nope" * 2' }, + }); + + expect(Object.keys(verdict.expression_errors)).toEqual(["bad"]); + }); + + test("validate_expressions reports the same verdict", async function () { + const table = await getClient().open_table("memory.superstore"); + const expressions = { good: '"Sales" * 2', bad: '"nope" * 2' }; + const validated = await table.validate_expressions(expressions); + expect(validated.expression_schema).toEqual({ good: "float" }); + expect(Object.keys(validated.errors)).toEqual(["bad"]); + expect(validated.expression_alias).toEqual(expressions); + }); + + test("view.expression_schema() answers from describe", async function () { + const table = await getClient().open_table("memory.superstore"); + const view = await table.view({ + columns: ["double"], + expressions: { double: '"Sales" * 2' }, + }); + + expect(await view.expression_schema()).toEqual({ double: "float" }); + await view.delete(); + }); +}); diff --git a/rust/perspective-python/perspective/tests/table/test_view_expression.py b/rust/perspective-python/perspective/tests/table/test_view_expression.py index 2849cf5cb0..8951f0e535 100644 --- a/rust/perspective-python/perspective/tests/table/test_view_expression.py +++ b/rust/perspective-python/perspective/tests/table/test_view_expression.py @@ -17,6 +17,8 @@ from datetime import date, datetime from time import mktime from perspective import PerspectiveError +import pytest + from .test_view import compare_delta import perspective as psp @@ -36,6 +38,61 @@ def test_table_validate_expressions_empty(self): assert validate["expression_alias"] == {} assert validate["errors"] == {} + def test_table_describe_valid(self): + table = Table({"a": [1, 2, 3, 4], "b": [5, 6, 7, 8]}) + config = dict( + columns=["a", "b", "x"], + group_by=["b"], + aggregates={"a": "avg"}, + expressions={"x": '"a" * 2'}, + ) + + verdict = table.describe(**config) + view = table.view(**config) + assert verdict == { + "expression_schema": {"x": "float"}, + "view_schema": view.schema(), + } + + view.delete() + + def test_table_describe_split_by_parity(self): + table = Table({"a": [1, 2, 3, 4], "b": ["x", "y", "x", "y"], "c": [1.5, 2.5, 3.5, 4.5]}) + for config in [ + dict(columns=["a", "c"], split_by=["b"]), + dict(columns=["a", "c"], split_by=["b"], aggregates={"a": "avg"}), + dict(columns=["a", "c"], group_by=["b"], split_by=["b"], aggregates={"a": "avg"}), + dict(columns=["a", "c"], split_by=["b"], group_rollup_mode="total", aggregates={"a": "avg"}), + ]: + verdict = table.describe(**config) + view = table.view(**config) + assert verdict["view_schema"] == view.schema(), config + view.delete() + + def test_table_describe_expression_errors(self): + table = Table({"a": [1, 2, 3, 4], "b": [5, 6, 7, 8]}) + verdict = table.describe( + columns=["a"], + expressions={"good": '"a" + 1', "bad": '"nope" + 1', "worse": "for () {}"}, + ) + + assert sorted(verdict.keys()) == ["expression_errors", "expression_schema"] + assert verdict["expression_schema"] == {"good": "float"} + assert sorted(verdict["expression_errors"].keys()) == ["bad", "worse"] + assert verdict["expression_errors"]["bad"] == { + "column": 0, + "error_message": 'Value Error - Input column "nope" does not exist.', + "line": 0, + } + + def test_table_describe_config_error(self): + table = Table({"a": [1, 2, 3, 4], "b": [5, 6, 7, 8]}) + verdict = table.describe(columns=["a", "nope"]) + assert list(verdict.keys()) == ["config_error"] + assert "nope" in verdict["config_error"] + with pytest.raises(Exception): + table.view(columns=["a", "nope"]) + def test_view_expression_schema_empty(self): table = Table({"a": [1, 2, 3, 4], "b": [5, 6, 7, 8]}) view = table.view() diff --git a/rust/perspective-python/perspective/tests/virtual_servers/test_duckdb.py b/rust/perspective-python/perspective/tests/virtual_servers/test_duckdb.py index 123a068412..dd8dbb99f5 100644 --- a/rust/perspective-python/perspective/tests/virtual_servers/test_duckdb.py +++ b/rust/perspective-python/perspective/tests/virtual_servers/test_duckdb.py @@ -639,6 +639,107 @@ def test_multi_column_sort(self, client): view.delete() +DESCRIBE_CONFIGS = [ + dict(columns=["Sales", "Quantity", "Region"]), + dict( + columns=["Sales", "Quantity"], + group_by=["Region"], + aggregates={"Sales": "sum", "Quantity": "avg"}, + ), + dict(columns=["Sales", "Quantity"], group_by=["Region"], group_rollup_mode="flat"), + dict(columns=["Sales"], group_by=["Region"], sort=[["Quantity", "desc"]]), + dict(columns=["Sales", "Quantity"], split_by=["Region"]), + dict(columns=["Sales", "Quantity"], split_by=["Region"], aggregates={"Quantity": "avg"}), + dict( + columns=["Sales", "Quantity"], + group_by=["Category"], + split_by=["Region"], + aggregates={"Sales": "sum", "Quantity": "avg"}, + ), + dict( + columns=["Sales"], + group_by=["Category"], + split_by=["Region"], + group_rollup_mode="flat", + ), + dict( + columns=["Sales"], + group_by=["Category"], + split_by=["Region"], + split_rollup_mode="rollup", + ), + dict(columns=["Sales", "Quantity"], group_rollup_mode="total", aggregates={"Quantity": "avg"}), + dict( + columns=["Sales", "Quantity"], + split_by=["Region"], + group_rollup_mode="total", + aggregates={"Quantity": "avg"}, + ), + dict(columns=["Sales", "double"], expressions={"double": '"Sales" * 2'}), + dict( + columns=["total"], + group_by=["Region"], + expressions={"total": '"Sales" + "Profit"'}, + aggregates={"total": "sum"}, + ), +] + + +class TestDuckDBDescribe: + @pytest.mark.parametrize("config", DESCRIBE_CONFIGS) + def test_parity_with_view(self, client, config): + table = client.open_table("memory.superstore") + verdict = table.describe(**config) + view = table.view(**config) + assert sorted(verdict.keys()) == ["expression_schema", "view_schema"] + assert verdict["view_schema"] == view.schema() + assert sorted(verdict["expression_schema"].keys()) == sorted( + config.get("expressions", {}).keys() + ) + view.delete() + + def test_attributes_a_bad_expression(self, client): + table = client.open_table("memory.superstore") + verdict = table.describe( + columns=["Sales", "good", "bad"], + expressions={"good": '"Sales" * 2', "bad": '"nope" * 2'}, + ) + + assert sorted(verdict.keys()) == ["expression_errors", "expression_schema"] + assert verdict["expression_schema"] == {"good": "float"} + assert list(verdict["expression_errors"].keys()) == ["bad"] + assert "nope" in verdict["expression_errors"]["bad"]["error_message"] + with pytest.raises(Exception): + table.view( + columns=["Sales", "good", "bad"], + expressions={"good": '"Sales" * 2', "bad": '"nope" * 2'}, + ) + + def test_unused_invalid_expression_is_still_rejected(self, client): + table = client.open_table("memory.superstore") + verdict = table.describe(columns=["Sales"], expressions={"bad": '"nope" * 2'}) + assert list(verdict["expression_errors"].keys()) == ["bad"] + + def test_bad_column_is_a_config_error(self, client): + table = client.open_table("memory.superstore") + verdict = table.describe(columns=["Sales", "nope"]) + assert list(verdict.keys()) == ["config_error"] + + def test_validate_expressions(self, client): + table = client.open_table("memory.superstore") + expressions = {"good": '"Sales" * 2', "bad": '"nope" * 2'} + validated = table.validate_expressions(expressions) + assert validated["expression_schema"] == {"good": "float"} + assert list(validated["errors"].keys()) == ["bad"] + assert validated["expression_alias"] == expressions + + def test_view_expression_schema(self, client): + table = client.open_table("memory.superstore") + view = table.view(columns=["double"], expressions={"double": '"Sales" * 2'}) + assert view.expression_schema() == {"double": "float"} + view.delete() + + class TestDuckDBExpressions: def test_simple_expression(self, client): table = client.open_table("memory.superstore") diff --git a/rust/perspective-python/perspective/tests/virtual_servers/test_polars.py b/rust/perspective-python/perspective/tests/virtual_servers/test_polars.py index 90127fbe7d..9d20747b0b 100644 --- a/rust/perspective-python/perspective/tests/virtual_servers/test_polars.py +++ b/rust/perspective-python/perspective/tests/virtual_servers/test_polars.py @@ -1010,6 +1010,34 @@ def test_split_by_only(self, client): ] view.delete() + def test_describe_parity_with_view(self, client): + table = client.open_table("superstore") + for config in [ + dict(columns=["Sales", "Quantity", "Region"]), + dict( + columns=["Sales", "Quantity"], + group_by=["Region"], + aggregates={"Sales": "sum", "Quantity": "avg"}, + ), + dict(columns=["Sales", "Quantity"], group_rollup_mode="total"), + dict(columns=["Sales", "double"], expressions={"double": '"Sales" * 2'}), + ]: + verdict = table.describe(**config) + view = table.view(**config) + assert sorted(verdict.keys()) == ["expression_schema", "view_schema"], config + assert verdict["view_schema"] == view.schema(), config + view.delete() + + def test_describe_reports_a_bad_expression(self, client): + table = client.open_table("superstore") + verdict = table.describe( + columns=["Sales", "double"], + expressions={"double": '"Sales" * 2', "bad": '"nope" * 2'}, + ) + + assert sorted(verdict.keys()) == ["expression_errors", "expression_schema"] + assert list(verdict["expression_errors"].keys()) == ["bad"] + def test_expressions_group_by_sort(self, client): table = client.open_table("superstore") view = table.view( diff --git a/rust/perspective-python/perspective/virtual_servers/__init__.py b/rust/perspective-python/perspective/virtual_servers/__init__.py index 550789b5e9..dcc4856c5a 100644 --- a/rust/perspective-python/perspective/virtual_servers/__init__.py +++ b/rust/perspective-python/perspective/virtual_servers/__init__.py @@ -104,12 +104,11 @@ def table_make_view(self, table_name, view_name, config): pass - def table_validate_expression(self, view_name, expression): + def table_describe(self, table_name, config): """ - [OPTIONAL] Given a temporary table `view_name`, validate the type of - a column expression string `expression`, or raise an error if the - expression is invalid. This is enabeld by `"expressions"` via - `get_features` and defaults to allow all expressions. + Validate a complete query configuration `config` against `table_name` + and report the schema `table_make_view` would create, without creating + it. """ pass @@ -140,3 +139,68 @@ def view_get_data(self, view_name, config, viewport, data): """ pass + + + +def describe_via_make_view(handler, table_name, config): + """A `table_describe` that builds a temporary table, reads its schema and + deletes it.""" + view_name = f"__psp_describe_{id(config)}__" + try: + handler.table_make_view(table_name, view_name, config) + except Exception as e: + return {"config_error": str(e)} + + try: + view_schema = handler.view_schema(view_name, config) + finally: + handler.view_delete(view_name) + + expression_schema = { + name: view_schema[name] + for name in config.get("expressions", {}) + if name in view_schema + } + + return {"expression_schema": expression_schema, "view_schema": view_schema} + + +def sql_table_describe(sql_builder, table_name, config, describe_query, schema=None): + """The `table_describe` flow shared by the SQL models.""" + expression_schema = {} + query = sql_builder.expressions_describe(table_name, config) + if query is not None: + try: + expression_schema = describe_query(query) + except Exception as error: + expression_errors = {} + for name, expression in config.get("expressions", {}).items(): + try: + single = describe_query( + sql_builder.expression_describe(table_name, expression) + ) + expression_schema[name] = next(iter(single.values())) + except Exception as e: + expression_errors[name] = { + "error_message": str(e), + "line": 0, + "column": 0, + } + + if not expression_errors: + return {"config_error": str(error)} + + return { + "expression_schema": expression_schema, + "expression_errors": expression_errors, + } + + view_schema = {} + query = sql_builder.table_describe(table_name, config, schema) + if query is not None: + try: + view_schema = describe_query(query) + except Exception as error: + return {"config_error": str(error)} + + return {"expression_schema": expression_schema, "view_schema": view_schema} diff --git a/rust/perspective-python/perspective/virtual_servers/clickhouse.py b/rust/perspective-python/perspective/virtual_servers/clickhouse.py index 909d6fb942..891e762326 100644 --- a/rust/perspective-python/perspective/virtual_servers/clickhouse.py +++ b/rust/perspective-python/perspective/virtual_servers/clickhouse.py @@ -15,7 +15,7 @@ from datetime import datetime import logging -from perspective.virtual_servers import VirtualServerHandler +from perspective.virtual_servers import VirtualServerHandler, sql_table_describe logger = logging.getLogger(__name__) @@ -200,10 +200,18 @@ def table_make_view(self, table_name, view_name, config): query = self.sql_builder.table_make_view(table_name, view_name, config, schema) run_query(self.db, query, execute=True) - def table_validate_expression(self, view_name, expression): - query = self.sql_builder.table_validate_expression(view_name, expression) - results = run_query(self.db, query) - return clickhouse_type_to_psp(results[0][1]) + def table_describe(self, table_name, config): + schema = self.table_schema(table_name) if config.get("windows") else None + return sql_table_describe( + self.sql_builder, table_name, config, self._describe_query, schema + ) + + def _describe_query(self, query): + return { + row[0]: clickhouse_type_to_psp(row[1]) + for row in run_query(self.db, query) + if not row[0].startswith("__") + } def view_delete(self, view_name): query = self.sql_builder.view_delete(view_name) diff --git a/rust/perspective-python/perspective/virtual_servers/duckdb.py b/rust/perspective-python/perspective/virtual_servers/duckdb.py index 074f9c2cbb..613d59a3b3 100644 --- a/rust/perspective-python/perspective/virtual_servers/duckdb.py +++ b/rust/perspective-python/perspective/virtual_servers/duckdb.py @@ -18,7 +18,7 @@ from datetime import datetime import logging -from perspective.virtual_servers import VirtualServerHandler +from perspective.virtual_servers import VirtualServerHandler, sql_table_describe logger = logging.getLogger(__name__) @@ -241,10 +241,18 @@ def table_make_view(self, table_name, view_name, config): query = self.sql_builder.table_make_view(table_name, view_name, config, schema) run_query(self.db, query, execute=True) - def table_validate_expression(self, view_name, expression): - query = self.sql_builder.table_validate_expression(view_name, expression) - results = run_query(self.db, query) - return duckdb_type_to_psp(results[0][1]) + def table_describe(self, table_name, config): + schema = self.table_schema(table_name) if config.get("windows") else None + return sql_table_describe( + self.sql_builder, table_name, config, self._describe_query, schema + ) + + def _describe_query(self, query): + return { + row[0]: duckdb_type_to_psp(row[1]) + for row in run_query(self.db, query) + if not row[0].startswith("__") + } def view_delete(self, view_name): query = self.sql_builder.view_delete(view_name) diff --git a/rust/perspective-python/perspective/virtual_servers/polars.py b/rust/perspective-python/perspective/virtual_servers/polars.py index 6d60ed9669..51858caf4a 100644 --- a/rust/perspective-python/perspective/virtual_servers/polars.py +++ b/rust/perspective-python/perspective/virtual_servers/polars.py @@ -139,17 +139,61 @@ def view_size(self, view_name): return self.views[view_name].height return self.table_size(view_name) - def table_validate_expression(self, table_name, expression): - df = self.tables.get(table_name) - if df is None: - return None - expr = parse_expression(expression) - result = df.select(expr.alias("__expr__")) - return polars_type_to_psp(result["__expr__"].dtype) + def table_describe(self, table_name, config): + empty = self.tables[table_name].clear() + expression_schema = {} + expression_errors = {} + for name, expr_str in config.get("expressions", {}).items(): + try: + expr = parse_expression(expr_str) + result = empty.select(expr.alias("__expr__")) + expression_schema[name] = polars_type_to_psp(result["__expr__"].dtype) + except Exception as e: + expression_errors[name] = { + "error_message": str(e), + "line": 0, + "column": 0, + } + + if expression_errors: + return { + "expression_schema": expression_schema, + "expression_errors": expression_errors, + } + + folded = dict(config) + folded["sort"] = [] + split_by = list(config.get("split_by", [])) + if split_by: + group_by = list(config.get("group_by", [])) + if not group_by and config.get("group_rollup_mode", "rollup") != "total": + folded["aggregates"] = { + c: "first" for c in config.get("columns", []) if c is not None + } + + folded["group_by"] = group_by + split_by + folded["split_by"] = [] + + try: + result = self._build_view(empty, folded) + except Exception as e: + return {"config_error": str(e)} + + return { + "expression_schema": expression_schema, + "view_schema": compute_view_schema(result), + } def table_make_view(self, table_name, view_name, config): start = datetime.now() - df = self.tables[table_name] + result = self._build_view(self.tables[table_name], config) + self.views[view_name] = result + self.view_schemas[view_name] = compute_view_schema(result) + logger.debug( + f"{datetime.now() - start} table_make_view {table_name} -> {view_name}" + ) + + def _build_view(self, df, config): group_by = config.get("group_by", []) columns = [c for c in config.get("columns", []) if c is not None] aggregates = config.get("aggregates", {}) @@ -207,11 +251,7 @@ def table_make_view(self, table_name, view_name, config): result = df.select(select_exprs) result = apply_sort_flat(result, sort, col_alias) - self.views[view_name] = result - self.view_schemas[view_name] = compute_view_schema(result) - logger.debug( - f"{datetime.now() - start} table_make_view {table_name} -> {view_name}" - ) + return result def view_delete(self, view_name): self.views.pop(view_name, None) diff --git a/rust/perspective-python/perspective/virtual_servers/postgres.py b/rust/perspective-python/perspective/virtual_servers/postgres.py index 7fa62c376a..849e3c5519 100644 --- a/rust/perspective-python/perspective/virtual_servers/postgres.py +++ b/rust/perspective-python/perspective/virtual_servers/postgres.py @@ -19,7 +19,7 @@ import psycopg import perspective -from perspective.virtual_servers import VirtualServerHandler +from perspective.virtual_servers import VirtualServerHandler, sql_table_describe logger = logging.getLogger(__name__) @@ -181,6 +181,7 @@ def __init__(self, conninfo): "row_id_expr": "ctid", "like_escape_clause": "\\", "regex_fn": "regexp_like", + "describe_template": "SELECT * FROM ({}) AS __psp_describe__ LIMIT 0", } ) @@ -279,14 +280,19 @@ def table_make_view(self, table_name, view_name, config): query = self.sql_builder.table_make_view(table_name, view_name, config, schema) run_query(self.db, query, execute=True) - def table_validate_expression(self, view_name, expression): - cur = run_query( - self.db, - f"SELECT {expression} FROM {view_name} LIMIT 0", - cursor=True, + def table_describe(self, table_name, config): + schema = self.table_schema(table_name) if config.get("windows") else None + return sql_table_describe( + self.sql_builder, table_name, config, self._describe_query, schema ) - return pg_oid_to_psp(cur.description[0].type_code, expression) + def _describe_query(self, query): + cur = run_query(self.db, query, cursor=True) + return { + d.name: pg_oid_to_psp(d.type_code, d.name) + for d in cur.description + if not d.name.startswith("__") + } def view_delete(self, view_name): query = self.sql_builder.view_delete(view_name) diff --git a/rust/perspective-python/src/client/client_async.rs b/rust/perspective-python/src/client/client_async.rs index af97722f6e..2cd9b1e190 100644 --- a/rust/perspective-python/src/client/client_async.rs +++ b/rust/perspective-python/src/client/client_async.rs @@ -16,11 +16,12 @@ use std::str::FromStr; use std::sync::Arc; use futures::FutureExt; +use perspective_client::config::ViewConfigUpdate; use perspective_client::proto::ListFlatten; use perspective_client::{ - Client, ColumnWindow, DeleteOptions, OnRemoveData, OnUpdateData, OnUpdateMode, OnUpdateOptions, - Table, TableData, TableInitOptions, TableReadFormat, TableRef, UpdateData, UpdateOptions, View, - ViewWindow, assert_table_api, assert_view_api, asyncfn, + Client, ColumnWindow, DeleteOptions, DescribeVerdict, OnRemoveData, OnUpdateData, OnUpdateMode, + OnUpdateOptions, Table, TableData, TableInitOptions, TableReadFormat, TableRef, UpdateData, + UpdateOptions, View, ViewWindow, assert_table_api, assert_view_api, asyncfn, }; use pyo3::exceptions::PyValueError; use pyo3::prelude::*; @@ -621,6 +622,32 @@ impl AsyncTable { Ok(()) } + /// Validate a complete view config against this table and report the + /// schema a [`View`] built from it would have, WITHOUT creating one. + /// + /// # Python Examples + /// + /// ```python + /// verdict = table.describe( + /// columns=["Sales", "margin"], + /// group_by=["Region"], + /// expressions={"margin": '"Profit" / "Sales"'}, + /// ) + /// + /// if "view_schema" in verdict: + /// view = table.view(**config) + /// ``` + #[pyo3(signature = (**kwargs))] + pub async fn describe(&self, kwargs: Option>) -> PyResult> { + let config: ViewConfigUpdate = kwargs + .map(|config| Python::attach(|py| depythonize(config.bind(py)))) + .transpose()? + .unwrap_or_default(); + + let verdict: DescribeVerdict = self.table.describe(config).await.into_pyerr()?.into(); + Python::attach(|py| Ok(pythonize::pythonize(py, &verdict)?.unbind())) + } + /// Validates the given expressions. pub async fn validate_expressions(&self, expressions: Py) -> PyResult> { let expressions = Python::attach(|py| depythonize(expressions.bind(py)))?; diff --git a/rust/perspective-python/src/client/client_sync.rs b/rust/perspective-python/src/client/client_sync.rs index fb5f182ca3..f6f9e97e8c 100644 --- a/rust/perspective-python/src/client/client_sync.rs +++ b/rust/perspective-python/src/client/client_sync.rs @@ -459,6 +459,14 @@ impl Table { table.schema().py_block_on(py) } + /// Validate a complete view config against this table and report the schema + /// a [`View`] built from it would have, WITHOUT creating one. + #[pyo3(signature = (**config))] + pub fn describe(&self, py: Python<'_>, config: Option>) -> PyResult> { + let table = self.0.clone(); + table.describe(config).py_block_on(py) + } + /// Validates the given expressions. pub fn validate_expressions( &self, diff --git a/rust/perspective-python/src/server/generic_sql_model.rs b/rust/perspective-python/src/server/generic_sql_model.rs index fe3ed6e501..2f1e9bb23f 100644 --- a/rust/perspective-python/src/server/generic_sql_model.rs +++ b/rust/perspective-python/src/server/generic_sql_model.rs @@ -62,12 +62,61 @@ impl PyGenericSQLVirtualServerModel { .map_err(|e| PyValueError::new_err(e.to_string())) } - pub fn table_validate_expression(&self, table_id: &str, expression: &str) -> PyResult { + /// The SQL query that plans every expression of `config` at once against a + /// table, one result column per expression alias, or `None` when the config + /// has no expressions. + pub fn expressions_describe( + &self, + table_id: &str, + config: Py, + ) -> PyResult> { + Python::attach(|py| { + let config: ViewConfig = pythonize::depythonize(config.bind(py)) + .map_err(|e| PyValueError::new_err(e.to_string()))?; + + self.inner + .expressions_describe(table_id, &config) + .map_err(|e| PyValueError::new_err(e.to_string())) + }) + } + + /// The SQL query that plans ONE expression against a table - for + /// attributing a failed `expressions_describe` to the expressions at fault. + pub fn expression_describe(&self, table_id: &str, expression: &str) -> PyResult { self.inner - .table_validate_expression(table_id, expression) + .expression_describe(table_id, expression) .map_err(|e| PyValueError::new_err(e.to_string())) } + /// The SQL query that plans the view `config` would build and reports its + /// result columns without materializing it (`split_by` folded into + /// `group_by`, no `PIVOT`), or `None` when the config selects no columns. + #[pyo3(signature = (table_id, config, schema=None))] + pub fn table_describe( + &self, + table_id: &str, + config: Py, + schema: Option>, + ) -> PyResult> { + Python::attach(|py| { + let config: ViewConfig = pythonize::depythonize(config.bind(py)) + .map_err(|e| PyValueError::new_err(e.to_string()))?; + + let schema = match &schema { + Some(schema) => { + self.parse_schema(schema.cast_bound::(py).map_err(|_| { + PyValueError::new_err("Schema must be a dict mapping column names to types") + })?)? + }, + None => IndexMap::new(), + }; + + self.inner + .table_describe(table_id, &config, &schema) + .map_err(|e| PyValueError::new_err(e.to_string())) + }) + } + pub fn view_delete(&self, view_id: &str) -> PyResult { self.inner .view_delete(view_id) diff --git a/rust/perspective-python/src/server/virtual_server_sync.rs b/rust/perspective-python/src/server/virtual_server_sync.rs index 8d952a6c69..8381dc0513 100644 --- a/rust/perspective-python/src/server/virtual_server_sync.rs +++ b/rust/perspective-python/src/server/virtual_server_sync.rs @@ -20,6 +20,7 @@ use perspective_client::virtual_server::{ Features, ResultExt, RowPathStyle, VirtualDataSlice, VirtualServer, VirtualServerFuture, VirtualServerHandler, }; +use perspective_client::{DescribeError, DescribeVerdict, Description}; use pyo3::exceptions::PyValueError; use pyo3::types::{ PyAnyMethods, PyBytes, PyDate, PyDict, PyDictMethods, PyList, PyListMethods, PyString, @@ -146,27 +147,38 @@ impl VirtualServerHandler for PyServerHandler { }) } - fn table_validate_expression( - &self, + fn table_describe( + &mut self, table_id: &str, - expression: &str, - ) -> VirtualServerFuture<'_, Result> { + config: &perspective_client::config::ViewConfig, + ) -> VirtualServerFuture<'_, Result, Self::Error>> { let handler = Python::attach(|py| self.0.clone_ref(py)); let table_id = table_id.to_string(); - let expression = expression.to_string(); + let config = config.clone(); Box::pin(async move { Python::attach(|py| { - let name = pyo3::intern!(py, "table_validate_expression"); - if handler.getattr(py, name).is_ok() { - Ok(handler - .call_method1(py, name, (&table_id, &expression))? - .cast_bound::(py)? - .extract::()?) - .map(|x| ColumnType::from_str(x.as_str()).unwrap()) - } else { - // TODO this should probably be an error. - Ok(ColumnType::Float) + let name = pyo3::intern!(py, "table_describe"); + if handler.getattr(py, name).is_err() { + return Err(PyValueError::new_err( + "`table_describe` is required of a virtual server handler", + )); } + + let result = handler.call_method1( + py, + name, + (&table_id, pythonize::pythonize(py, &config)?), + )?; + + let verdict: DescribeVerdict = + pythonize::depythonize(result.bind(py)).map_err(|e| { + PyValueError::new_err(format!( + "`table_describe` must return a describe verdict dict: {e}" + )) + })?; + + Result::::try_from(verdict) + .map_err(|e| PyValueError::new_err(e.to_string())) }) }) } diff --git a/rust/perspective-server/cpp/perspective/patches/fix_arrow_libtool.patch b/rust/perspective-server/cpp/perspective/patches/fix_arrow_libtool.patch index 81a16bffd7..a2ed4edfe9 100644 --- a/rust/perspective-server/cpp/perspective/patches/fix_arrow_libtool.patch +++ b/rust/perspective-server/cpp/perspective/patches/fix_arrow_libtool.patch @@ -51,8 +51,9 @@ index 692efa78376f4..eb94563afcf77 100644 execute_process(COMMAND ${LIBTOOL_MACOS} -V OUTPUT_VARIABLE LIBTOOL_V_OUTPUT OUTPUT_STRIP_TRAILING_WHITESPACE) - if(NOT "${LIBTOOL_V_OUTPUT}" MATCHES ".*cctools-([0-9.]+).*") +- if(NOT "${LIBTOOL_V_OUTPUT}" MATCHES ".*cctools-([0-9.]+).*") - message(FATAL_ERROR "libtool found appears to be the incompatible GNU libtool: ${LIBTOOL_MACOS}" ++ if(NOT "${LIBTOOL_V_OUTPUT}" MATCHES ".*cctools(_ld)?-([0-9.]+).*") + message(FATAL_ERROR "libtool found appears not to be Apple's libtool: ${LIBTOOL_MACOS}" ) endif() diff --git a/rust/perspective-server/cpp/perspective/src/cpp/server.cpp b/rust/perspective-server/cpp/perspective/src/cpp/server.cpp index 0ba3ec4bea..bd3d61c62a 100644 --- a/rust/perspective-server/cpp/perspective/src/cpp/server.cpp +++ b/rust/perspective-server/cpp/perspective/src/cpp/server.cpp @@ -1256,7 +1256,6 @@ needs_poll(const proto::Request::ClientReqCase proto_case) { case ReqCase::kTableSizeReq: case ReqCase::kTableSchemaReq: case ReqCase::kTableMakePortReq: - case ReqCase::kTableValidateExprReq: case ReqCase::kMakeTableReq: case ReqCase::kViewDimensionsReq: case ReqCase::kViewToColumnsStringReq: @@ -1292,6 +1291,7 @@ needs_poll(const proto::Request::ClientReqCase proto_case) { case ReqCase::kServerSystemInfoReq: case ReqCase::kGetFeaturesReq: case ReqCase::kMakeJoinTableReq: + case ReqCase::kTableDescribeReq: return false; case proto::Request::CLIENT_REQ_NOT_SET: throw std::runtime_error("Unhandled request type 2"); @@ -1307,7 +1307,7 @@ entity_type_is_table(const proto::Request::ClientReqCase proto_case) { case ReqCase::kTableSizeReq: case ReqCase::kTableSchemaReq: case ReqCase::kTableMakePortReq: - case ReqCase::kTableValidateExprReq: + case ReqCase::kTableDescribeReq: case ReqCase::kMakeTableReq: case ReqCase::kTableOnDeleteReq: case ReqCase::kTableRemoveReq: @@ -1559,6 +1559,539 @@ coerce_to(const t_dtype dtype, const A& val) { } } +ProtoServer::BuiltViewConfig +ProtoServer::build_view_config( + const std::shared_ptr& table, const proto::ViewConfig& cfg +) { + auto schema = + std::make_shared(table->get_gnode()->get_output_schema()); + + const auto& group_by = cfg.group_by(); + std::vector row_pivots{ + group_by.begin(), group_by.end() + }; + + const auto& split_by = cfg.split_by(); + std::vector column_pivots{ + split_by.begin(), split_by.end() + }; + + const auto& aggs = cfg.aggregates(); + tsl::ordered_map> aggregates; + for (const auto& [col_name, agg_list] : aggs) { + aggregates[col_name] = std::vector(); + for (const auto& agg : agg_list.aggregations()) { + aggregates[col_name].push_back(agg); + } + } + + const auto& sorts = cfg.sort(); + std::vector sortby; + std::vector> sort_str; + for (const auto& sort : sorts) { + const char* column_sort = sort_op_str_from_proto(sort.op()); + sort_str.push_back({sort.column(), column_sort}); + } + + bool column_only = false; + bool is_total = + cfg.has_group_rollup_mode() ? cfg.group_rollup_mode() == 2 : false; + + // make sure that primary keys are created for column-only views + if (row_pivots.empty() && !column_pivots.empty()) { + row_pivots.emplace_back("psp_okey"); + column_only = true; + } + + std::vector> expressions; + auto exprs = parse_expression_strings(cfg.expressions()); + + std::vector>>> + legacy_exprs; + + legacy_exprs.resize(1); + for (const auto& expr : exprs) { + legacy_exprs[0] = { + expr.expression_alias, + expr.expression, + expr.parse_expression_string, + std::vector>{ + expr.column_id_map.begin(), expr.column_id_map.end() + } + }; + + // Validate these expression, creating is not the same thing! + const auto& res = table->validate_expressions(legacy_exprs); + if (!res.get_expression_errors().empty()) { + // TODO unify error reporting - this works differently than + // `validate_expressions()`. In this case there is + // guaranteed to only be one ... + PSP_COMPLAIN_AND_ABORT(res.get_expression_errors() + .at(expr.expression_alias) + .m_error_message); + } + + const auto& gnode = table->get_gnode(); + auto column_id_map = + std::vector>( + expr.column_id_map.begin(), expr.column_id_map.end() + ); + + auto expr_vocab = gnode->get_expression_vocab(); + t_expression_vocab& expression_vocab = *expr_vocab; + auto expression_regex_mapping = + gnode->get_expression_regex_mapping(); + t_regex_mapping& regex_mapping = *expression_regex_mapping; + + std::shared_ptr computed_expression = + m_computed_expression_parser.precompute( + expr.expression_alias, + expr.expression, + expr.parse_expression_string, + column_id_map, + gnode->get_table_sptr(), + gnode->get_pkey_map(), + schema, + expression_vocab, + regex_mapping + ); + + auto dtype = computed_expression->get_dtype(); + + schema->add_column(expr.expression_alias, dtype); + expressions.push_back(std::make_shared( + expr.expression_alias, + expr.expression, + expr.parse_expression_string, + column_id_map, + dtype + )); + } + + std::vector windows; + windows.reserve(cfg.windows_size()); + const t_schema& table_schema = table->get_schema(); + + // `windows` is a proto map keyed by output alias; iterate in + // sorted-name order so output column registration (and thus + // any error precedence) is deterministic. + std::vector window_names; + window_names.reserve(cfg.windows_size()); + for (const auto& it : cfg.windows()) { + window_names.push_back(it.first); + } + std::sort(window_names.begin(), window_names.end()); + for (const auto& name : window_names) { + const auto& w = cfg.windows().at(name); + if (name.empty()) { + PSP_COMPLAIN_AND_ABORT("Window `name` must not be empty"); + } + + if (schema->has_column(name)) { + PSP_COMPLAIN_AND_ABORT( + "Window `name` collides with an existing column: " + + name + ); + } + + if (!schema->has_column(w.source())) { + PSP_COMPLAIN_AND_ABORT( + "Window `source` column not found: " + w.source() + ); + } + + // `order_by`/`partition_by` must be real `Table` columns - + // the window engine reads them from the gnode master table, + // where expression aliases do not exist. An OMITTED + // `order_by` takes natural (primary key) order. + if (w.has_order_by() + && !table_schema.has_column(w.order_by().column())) { + PSP_COMPLAIN_AND_ABORT( + "Window `order_by` must be a `Table` column: " + + w.order_by().column() + ); + } + + for (const auto& p : w.partition_by()) { + if (!table_schema.has_column(p)) { + PSP_COMPLAIN_AND_ABORT( + "Window `partition_by` must be a `Table` column: " + + p + ); + } + } + + static const std::unordered_map + WINDOW_OPS{ + {"sum", t_window_op::WINDOW_OP_SUM}, + {"avg", t_window_op::WINDOW_OP_AVG}, + {"count", t_window_op::WINDOW_OP_COUNT}, + {"min", t_window_op::WINDOW_OP_MIN}, + {"max", t_window_op::WINDOW_OP_MAX}, + {"stddev", t_window_op::WINDOW_OP_STDDEV}, + {"var", t_window_op::WINDOW_OP_VAR}, + {"first", t_window_op::WINDOW_OP_FIRST}, + {"last", t_window_op::WINDOW_OP_LAST}, + {"lag", t_window_op::WINDOW_OP_LAG}, + {"lead", t_window_op::WINDOW_OP_LEAD}, + {"diff", t_window_op::WINDOW_OP_DIFF}, + {"rate", t_window_op::WINDOW_OP_RATE}, + {"ema", t_window_op::WINDOW_OP_EMA}, + }; + + const auto op_entry = WINDOW_OPS.find(w.op()); + if (op_entry == WINDOW_OPS.end()) { + PSP_COMPLAIN_AND_ABORT( + "Window `op` not implemented in this build: " + w.op() + ); + } + + t_window_op op = op_entry->second; + + // An OMITTED frame means cumulative for aggregating + // ops - the initializer below IS that default. + t_window_frame_type frame_type = + t_window_frame_type::WINDOW_FRAME_CUMULATIVE; + t_uindex frame_rows = 0; + double frame_range = 0; + bool has_frame = true; + switch (w.frame_case()) { + case proto::WindowSpec::kRows: + frame_type = t_window_frame_type::WINDOW_FRAME_ROWS; + frame_rows = w.rows(); + break; + case proto::WindowSpec::kCumulative: + frame_type = + t_window_frame_type::WINDOW_FRAME_CUMULATIVE; + break; + case proto::WindowSpec::kRange: { + frame_type = t_window_frame_type::WINDOW_FRAME_RANGE; + frame_range = w.range(); + if (!(frame_range > 0)) { + PSP_COMPLAIN_AND_ABORT( + "Window `range` must be a positive interval" + ); + } + + // Interval arithmetic is defined on the order + // column's raw units (ms for datetime, days for + // date) - the natural-order fallback has no units, + // so `range` requires an explicit `order_by`. + if (!w.has_order_by()) { + PSP_COMPLAIN_AND_ABORT( + "Window `range` frames require an explicit " + "`order_by`" + ); + } + + t_dtype order_dtype = + table_schema.get_dtype(w.order_by().column()); + switch (order_dtype) { + case DTYPE_INT8: + case DTYPE_INT16: + case DTYPE_INT32: + case DTYPE_INT64: + case DTYPE_UINT8: + case DTYPE_UINT16: + case DTYPE_UINT32: + case DTYPE_UINT64: + case DTYPE_FLOAT32: + case DTYPE_FLOAT64: + case DTYPE_TIME: + case DTYPE_DATE: + break; + default: + PSP_COMPLAIN_AND_ABORT( + "Window `range` frames require a numeric " + "or temporal `order_by` column: " + + w.order_by().column() + ); + } + } break; + default: + has_frame = false; + break; + } + + switch (op) { + case t_window_op::WINDOW_OP_LAG: + case t_window_op::WINDOW_OP_LEAD: + case t_window_op::WINDOW_OP_DIFF: + if (has_frame) { + PSP_COMPLAIN_AND_ABORT( + "Window `frame` is not applicable to " + "`lag`/`lead`/`diff` (use `offset`)" + ); + } + break; + case t_window_op::WINDOW_OP_RATE: + if (frame_type + != t_window_frame_type::WINDOW_FRAME_RANGE + || !has_frame) { + PSP_COMPLAIN_AND_ABORT( + "Window `rate` requires a `range` frame" + ); + } + break; + case t_window_op::WINDOW_OP_EMA: + if (has_frame + && frame_type + != t_window_frame_type:: + WINDOW_FRAME_CUMULATIVE) { + PSP_COMPLAIN_AND_ABORT( + "Window `ema` is cumulative; it does not " + "accept a `rows` or `range` frame" + ); + } + + if (!w.has_alpha() || !(w.alpha() > 0) + || w.alpha() > 1) { + PSP_COMPLAIN_AND_ABORT( + "Window `ema` requires `alpha` in (0, 1]" + ); + } + break; + default: + break; + } + + if (!t_window_engine::is_implemented(op, frame_type)) { + PSP_COMPLAIN_AND_ABORT( + "Window op/frame combination not implemented" + ); + } + + t_dtype source_dtype = schema->get_dtype(w.source()); + t_dtype dtype = + t_window_engine::resolve_dtype(op, source_dtype); + if (dtype == DTYPE_NONE) { + PSP_COMPLAIN_AND_ABORT( + "Window op requires a numeric `source` column: " + + w.source() + ); + } + + t_window_spec spec; + spec.m_name = name; + spec.m_source = w.source(); + + // Empty `m_order_by` = natural (primary key) order; the + // engine's comparator degenerates to its pkey tiebreak when + // every order key is absent. + spec.m_order_by = + w.has_order_by() ? w.order_by().column() : ""; + spec.m_order_desc = + w.has_order_by() && w.order_by().desc(); + spec.m_partition_by = { + w.partition_by().begin(), w.partition_by().end() + }; + spec.m_op = op; + spec.m_frame_type = frame_type; + spec.m_frame_rows = frame_rows; + spec.m_frame_range = frame_range; + spec.m_offset = w.has_offset() ? w.offset() : 1; + spec.m_alpha = w.has_alpha() ? w.alpha() : 0; + spec.m_dtype = dtype; + + schema->add_column(spec.m_name, dtype); + windows.push_back(std::move(spec)); + } + + t_vocab vocab; + vocab.init(false); + std::vector< + std::tuple>> + filter; + filter.reserve(cfg.filter().size()); + + for (const auto& f : cfg.filter()) { + for (const auto& arg : f.value()) { + switch (arg.scalar_case()) { + case proto::Scalar::kString: { +#ifdef PSP_SSO_SCALAR + if (!t_tscalar::can_store_inplace(arg.string())) { + vocab.get_interned(arg.string()); + } +#else + vocab.get_interned(arg.string()); +#endif + break; + } + case proto::Scalar::kBool: + case proto::Scalar::kFloat: + case proto::Scalar::kNull: + case proto::Scalar::SCALAR_NOT_SET: + break; + } + } + } + + for (const auto& f : cfg.filter()) { + std::vector args; + args.reserve(f.value().size()); + for (const auto& arg : f.value()) { + t_tscalar a; + a.clear(); + switch (arg.scalar_case()) { + case proto::Scalar::kBool: { + a.set(arg.bool_()); + args.push_back(a); + break; + } + case proto::Scalar::kFloat: { + a = coerce_to( + schema->get_dtype(f.column()), arg.float_() + ); + + args.push_back(a); + break; + } + case proto::Scalar::kString: { + if (!schema->has_column(f.column())) { + PSP_COMPLAIN_AND_ABORT( + "Filter column not in schema: " + f.column() + ); + } + +#ifdef PSP_SSO_SCALAR + if (!t_tscalar::can_store_inplace(arg.string())) { +#endif + a = coerce_to( + schema->get_dtype(f.column()), + vocab.unintern_c( + vocab.get_interned(arg.string()) + ) + ); +#ifdef PSP_SSO_SCALAR + } else { + + a = coerce_to( + schema->get_dtype(f.column()), + arg.string().c_str() + ); + } +#endif + args.push_back(a); + break; + } + case proto::Scalar::kNull: + a.set(t_none()); + args.push_back(a); + break; + case proto::Scalar::SCALAR_NOT_SET: + PSP_COMPLAIN_AND_ABORT( + "Filter scalar type not implemented: " + + std::to_string(arg.scalar_case()) + ) + break; + } + } + + filter.emplace_back(f.column(), f.op(), args); + } + + const auto& cols = cfg.columns(); + std::vector columns; + if (cols.has_columns()) { + columns = { + cols.columns().columns().begin(), + cols.columns().columns().end() + }; + } else { + columns = table->get_column_names(); + for (const auto& f : expressions) { + columns.push_back(f->get_expression_alias()); + } + for (const auto& w : windows) { + columns.push_back(w.m_name); + } + } + + LOG_DEBUG( + "Creating view config with \n" + << "row_pivots: " << row_pivots << '\n' + << "column_pivots: " << column_pivots + << '\n' + // << "aggregates: " << aggregates << '\n' + << "columns: " << columns + << '\n' + // << "filter: " << filter << '\n' + << "sort_str: " << sort_str << '\n' + << "expressions: " << expressions << '\n' + << "column_only: " << column_only << '\n' + ); + + std::string filter_op; + switch (cfg.filter_op()) { + case proto::ViewConfig_FilterReducer:: + ViewConfig_FilterReducer_OR: + filter_op = "or"; + break; + case proto::ViewConfig_FilterReducer:: + ViewConfig_FilterReducer_AND: + default: + filter_op = "and"; + break; + } + + LOG_DEBUG("FILTER_OP: " << filter_op); + + bool leaves_only = + cfg.has_group_rollup_mode() ? cfg.group_rollup_mode() == 1 : false; + bool total_only = + cfg.has_group_rollup_mode() ? cfg.group_rollup_mode() == 2 : false; + bool split_rollup = cfg.has_split_rollup_mode() + && cfg.split_rollup_mode() + == proto::SplitRollupMode::SPLIT_ROLLUP_MODE_ROLLUP; + + auto config = std::make_shared( + vocab, + row_pivots, + column_pivots, + aggregates, + columns, + filter, + sort_str, + expressions, + filter_op, + column_only, + leaves_only, + total_only, + windows, + split_rollup + ); + config->init(schema); + + if (cfg.has_group_by_depth()) { + config->set_row_pivot_depth(cfg.group_by_depth()); + } + + std::uint32_t sides; + + if (!group_by.empty() || !split_by.empty()) { + if (!split_by.empty()) { + sides = 2; + } else { + sides = 1; + } + } else if (total_only) { + sides = 1; + } else { + sides = 0; + } + + bool is_unit_context = table->get_index().empty() && sides == 0 + && row_pivots.empty() && column_pivots.empty() + && aggregates.empty() && columns.empty() && sort_str.empty() + && cfg.expressions().empty() && cfg.windows().empty(); + + return BuiltViewConfig{schema, config, sides, is_unit_context}; +} + std::vector> ProtoServer::_handle_request(std::uint32_t client_id, Request&& req) { std::vector> proto_resp; @@ -2171,17 +2704,13 @@ ProtoServer::_handle_request(std::uint32_t client_id, Request&& req) { push_resp(std::move(resp)); break; } - case proto::Request::kTableValidateExprReq: { + case proto::Request::kTableDescribeReq: { auto table = m_resources.get_table(req.entity_id()); - const auto& r = req.table_validate_expr_req(); - - const auto& col_with_expr = r.column_to_expr(); - const auto& exprs = parse_expression_strings(col_with_expr); - - // TODO: validate the expression, mocked out for now + const auto& cfg = req.table_describe_req().config(); proto::Response resp; - auto* validate_expr = resp.mutable_table_validate_expr_resp(); + auto* out = resp.mutable_table_describe_resp(); + const auto& exprs = parse_expression_strings(cfg.expressions()); std::vectorvalidate_expressions(legacy_exprs); - - std::vector> schema; for (const auto& [col, val] : res.get_expression_schema()) { - schema.emplace_back( - col, dtype_to_column_type(str_to_dtype(val)) - ); - } + (*out->mutable_expression_schema())[col] = + dtype_to_column_type(str_to_dtype(val)); + } + + const auto& errors = res.get_expression_errors(); + if (!errors.empty()) { + for (const auto& [col_name, err] : errors) { + proto::ExpressionError proto_err; + *proto_err.mutable_error_message() = err.m_error_message; + proto_err.set_column(err.m_column); + proto_err.set_line(err.m_line); + (*out->mutable_expression_errors())[col_name] = + std::move(proto_err); + } - for (auto&& [col_name, col_type] : schema) { - (*validate_expr->mutable_expression_schema())[col_name] = - col_type; + push_resp(std::move(resp)); + break; } - std::vector> - errors; - for (const auto& [col_name, err] : res.get_expression_errors()) { - proto::TableValidateExprResp_ExprValidationError proto_err; - *proto_err.mutable_error_message() = err.m_error_message; - proto_err.set_column(err.m_column); - proto_err.set_line(err.m_line); - (*validate_expr->mutable_errors())[col_name] = - std::move(proto_err); - } + try { + auto built = build_view_config(table, cfg); + auto* view = out->mutable_view(); + const auto schema = describe_view_schema( + *built.config, *built.schema, built.sides > 0 + ); - for (const auto& [col_name, col_expr] : r.column_to_expr()) { - (*validate_expr->mutable_expression_alias())[col_name] = - col_expr; + for (const auto& [name, type_str] : schema) { + (*view->mutable_schema())[name] = + dtype_to_column_type(str_to_dtype(type_str)); + } + } catch (const PerspectiveException& e) { + out->set_config_error(e.what()); + } catch (const std::exception& e) { + out->set_config_error(e.what()); } push_resp(std::move(resp)); @@ -2363,534 +2898,12 @@ ProtoServer::_handle_request(std::uint32_t client_id, Request&& req) { } case proto::Request::kTableMakeViewReq: { auto table = m_resources.get_table(req.entity_id()); - auto schema = std::make_shared( - table->get_gnode()->get_output_schema() - ); const auto& r = req.table_make_view_req(); - const auto& cfg = r.config(); - - const auto& group_by = cfg.group_by(); - std::vector row_pivots{ - group_by.begin(), group_by.end() - }; - - const auto& split_by = cfg.split_by(); - std::vector column_pivots{ - split_by.begin(), split_by.end() - }; - - const auto& aggs = cfg.aggregates(); - tsl::ordered_map> aggregates; - for (const auto& [col_name, agg_list] : aggs) { - aggregates[col_name] = std::vector(); - for (const auto& agg : agg_list.aggregations()) { - aggregates[col_name].push_back(agg); - } - } - - const auto& sorts = cfg.sort(); - std::vector sortby; - std::vector> sort_str; - for (const auto& sort : sorts) { - const char* column_sort = sort_op_str_from_proto(sort.op()); - sort_str.push_back({sort.column(), column_sort}); - } - - bool column_only = false; - bool is_total = - cfg.has_group_rollup_mode() ? cfg.group_rollup_mode() == 2 : false; - - // make sure that primary keys are created for column-only views - if (row_pivots.empty() && !column_pivots.empty()) { - row_pivots.emplace_back("psp_okey"); - column_only = true; - } - - std::vector> expressions; - auto exprs = parse_expression_strings(cfg.expressions()); - - std::vector>>> - legacy_exprs; - - legacy_exprs.resize(1); - for (const auto& expr : exprs) { - legacy_exprs[0] = { - expr.expression_alias, - expr.expression, - expr.parse_expression_string, - std::vector>{ - expr.column_id_map.begin(), expr.column_id_map.end() - } - }; - - // Validate these expression, creating is not the same thing! - const auto& res = table->validate_expressions(legacy_exprs); - if (!res.get_expression_errors().empty()) { - // TODO unify error reporting - this works differently than - // `validate_expressions()`. In this case there is - // guaranteed to only be one ... - PSP_COMPLAIN_AND_ABORT(res.get_expression_errors() - .at(expr.expression_alias) - .m_error_message); - } - - const auto& gnode = table->get_gnode(); - auto column_id_map = - std::vector>( - expr.column_id_map.begin(), expr.column_id_map.end() - ); - - auto expr_vocab = gnode->get_expression_vocab(); - t_expression_vocab& expression_vocab = *expr_vocab; - auto expression_regex_mapping = - gnode->get_expression_regex_mapping(); - t_regex_mapping& regex_mapping = *expression_regex_mapping; - - std::shared_ptr computed_expression = - m_computed_expression_parser.precompute( - expr.expression_alias, - expr.expression, - expr.parse_expression_string, - column_id_map, - gnode->get_table_sptr(), - gnode->get_pkey_map(), - schema, - expression_vocab, - regex_mapping - ); - - auto dtype = computed_expression->get_dtype(); - - schema->add_column(expr.expression_alias, dtype); - expressions.push_back(std::make_shared( - expr.expression_alias, - expr.expression, - expr.parse_expression_string, - column_id_map, - dtype - )); - } - - std::vector windows; - windows.reserve(cfg.windows_size()); - const t_schema& table_schema = table->get_schema(); - - // `windows` is a proto map keyed by output alias; iterate in - // sorted-name order so output column registration (and thus - // any error precedence) is deterministic. - std::vector window_names; - window_names.reserve(cfg.windows_size()); - for (const auto& it : cfg.windows()) { - window_names.push_back(it.first); - } - std::sort(window_names.begin(), window_names.end()); - for (const auto& name : window_names) { - const auto& w = cfg.windows().at(name); - if (name.empty()) { - PSP_COMPLAIN_AND_ABORT("Window `name` must not be empty"); - } - - if (schema->has_column(name)) { - PSP_COMPLAIN_AND_ABORT( - "Window `name` collides with an existing column: " - + name - ); - } - - if (!schema->has_column(w.source())) { - PSP_COMPLAIN_AND_ABORT( - "Window `source` column not found: " + w.source() - ); - } - - // `order_by`/`partition_by` must be real `Table` columns - - // the window engine reads them from the gnode master table, - // where expression aliases do not exist. An OMITTED - // `order_by` takes natural (primary key) order. - if (w.has_order_by() - && !table_schema.has_column(w.order_by().column())) { - PSP_COMPLAIN_AND_ABORT( - "Window `order_by` must be a `Table` column: " - + w.order_by().column() - ); - } - - for (const auto& p : w.partition_by()) { - if (!table_schema.has_column(p)) { - PSP_COMPLAIN_AND_ABORT( - "Window `partition_by` must be a `Table` column: " - + p - ); - } - } - - static const std::unordered_map - WINDOW_OPS{ - {"sum", t_window_op::WINDOW_OP_SUM}, - {"avg", t_window_op::WINDOW_OP_AVG}, - {"count", t_window_op::WINDOW_OP_COUNT}, - {"min", t_window_op::WINDOW_OP_MIN}, - {"max", t_window_op::WINDOW_OP_MAX}, - {"stddev", t_window_op::WINDOW_OP_STDDEV}, - {"var", t_window_op::WINDOW_OP_VAR}, - {"first", t_window_op::WINDOW_OP_FIRST}, - {"last", t_window_op::WINDOW_OP_LAST}, - {"lag", t_window_op::WINDOW_OP_LAG}, - {"lead", t_window_op::WINDOW_OP_LEAD}, - {"diff", t_window_op::WINDOW_OP_DIFF}, - {"rate", t_window_op::WINDOW_OP_RATE}, - {"ema", t_window_op::WINDOW_OP_EMA}, - }; - - const auto op_entry = WINDOW_OPS.find(w.op()); - if (op_entry == WINDOW_OPS.end()) { - PSP_COMPLAIN_AND_ABORT( - "Window `op` not implemented in this build: " + w.op() - ); - } - - t_window_op op = op_entry->second; - - // An OMITTED frame means cumulative for aggregating - // ops - the initializer below IS that default. - t_window_frame_type frame_type = - t_window_frame_type::WINDOW_FRAME_CUMULATIVE; - t_uindex frame_rows = 0; - double frame_range = 0; - bool has_frame = true; - switch (w.frame_case()) { - case proto::WindowSpec::kRows: - frame_type = t_window_frame_type::WINDOW_FRAME_ROWS; - frame_rows = w.rows(); - break; - case proto::WindowSpec::kCumulative: - frame_type = - t_window_frame_type::WINDOW_FRAME_CUMULATIVE; - break; - case proto::WindowSpec::kRange: { - frame_type = t_window_frame_type::WINDOW_FRAME_RANGE; - frame_range = w.range(); - if (!(frame_range > 0)) { - PSP_COMPLAIN_AND_ABORT( - "Window `range` must be a positive interval" - ); - } - - // Interval arithmetic is defined on the order - // column's raw units (ms for datetime, days for - // date) - the natural-order fallback has no units, - // so `range` requires an explicit `order_by`. - if (!w.has_order_by()) { - PSP_COMPLAIN_AND_ABORT( - "Window `range` frames require an explicit " - "`order_by`" - ); - } - - t_dtype order_dtype = - table_schema.get_dtype(w.order_by().column()); - switch (order_dtype) { - case DTYPE_INT8: - case DTYPE_INT16: - case DTYPE_INT32: - case DTYPE_INT64: - case DTYPE_UINT8: - case DTYPE_UINT16: - case DTYPE_UINT32: - case DTYPE_UINT64: - case DTYPE_FLOAT32: - case DTYPE_FLOAT64: - case DTYPE_TIME: - case DTYPE_DATE: - break; - default: - PSP_COMPLAIN_AND_ABORT( - "Window `range` frames require a numeric " - "or temporal `order_by` column: " - + w.order_by().column() - ); - } - } break; - default: - has_frame = false; - break; - } - - switch (op) { - case t_window_op::WINDOW_OP_LAG: - case t_window_op::WINDOW_OP_LEAD: - case t_window_op::WINDOW_OP_DIFF: - if (has_frame) { - PSP_COMPLAIN_AND_ABORT( - "Window `frame` is not applicable to " - "`lag`/`lead`/`diff` (use `offset`)" - ); - } - break; - case t_window_op::WINDOW_OP_RATE: - if (frame_type - != t_window_frame_type::WINDOW_FRAME_RANGE - || !has_frame) { - PSP_COMPLAIN_AND_ABORT( - "Window `rate` requires a `range` frame" - ); - } - break; - case t_window_op::WINDOW_OP_EMA: - if (has_frame - && frame_type - != t_window_frame_type:: - WINDOW_FRAME_CUMULATIVE) { - PSP_COMPLAIN_AND_ABORT( - "Window `ema` is cumulative; it does not " - "accept a `rows` or `range` frame" - ); - } - - if (!w.has_alpha() || !(w.alpha() > 0) - || w.alpha() > 1) { - PSP_COMPLAIN_AND_ABORT( - "Window `ema` requires `alpha` in (0, 1]" - ); - } - break; - default: - break; - } - - if (!t_window_engine::is_implemented(op, frame_type)) { - PSP_COMPLAIN_AND_ABORT( - "Window op/frame combination not implemented" - ); - } - - t_dtype source_dtype = schema->get_dtype(w.source()); - t_dtype dtype = - t_window_engine::resolve_dtype(op, source_dtype); - if (dtype == DTYPE_NONE) { - PSP_COMPLAIN_AND_ABORT( - "Window op requires a numeric `source` column: " - + w.source() - ); - } - - t_window_spec spec; - spec.m_name = name; - spec.m_source = w.source(); - - // Empty `m_order_by` = natural (primary key) order; the - // engine's comparator degenerates to its pkey tiebreak when - // every order key is absent. - spec.m_order_by = - w.has_order_by() ? w.order_by().column() : ""; - spec.m_order_desc = - w.has_order_by() && w.order_by().desc(); - spec.m_partition_by = { - w.partition_by().begin(), w.partition_by().end() - }; - spec.m_op = op; - spec.m_frame_type = frame_type; - spec.m_frame_rows = frame_rows; - spec.m_frame_range = frame_range; - spec.m_offset = w.has_offset() ? w.offset() : 1; - spec.m_alpha = w.has_alpha() ? w.alpha() : 0; - spec.m_dtype = dtype; - - schema->add_column(spec.m_name, dtype); - windows.push_back(std::move(spec)); - } - - t_vocab vocab; - vocab.init(false); - std::vector< - std::tuple>> - filter; - filter.reserve(cfg.filter().size()); - - for (const auto& f : cfg.filter()) { - for (const auto& arg : f.value()) { - switch (arg.scalar_case()) { - case proto::Scalar::kString: { -#ifdef PSP_SSO_SCALAR - if (!t_tscalar::can_store_inplace(arg.string())) { - vocab.get_interned(arg.string()); - } -#else - vocab.get_interned(arg.string()); -#endif - break; - } - case proto::Scalar::kBool: - case proto::Scalar::kFloat: - case proto::Scalar::kNull: - case proto::Scalar::SCALAR_NOT_SET: - break; - } - } - } - - for (const auto& f : cfg.filter()) { - std::vector args; - args.reserve(f.value().size()); - for (const auto& arg : f.value()) { - t_tscalar a; - a.clear(); - switch (arg.scalar_case()) { - case proto::Scalar::kBool: { - a.set(arg.bool_()); - args.push_back(a); - break; - } - case proto::Scalar::kFloat: { - a = coerce_to( - schema->get_dtype(f.column()), arg.float_() - ); - - args.push_back(a); - break; - } - case proto::Scalar::kString: { - if (!schema->has_column(f.column())) { - PSP_COMPLAIN_AND_ABORT( - "Filter column not in schema: " + f.column() - ); - } - -#ifdef PSP_SSO_SCALAR - if (!t_tscalar::can_store_inplace(arg.string())) { -#endif - a = coerce_to( - schema->get_dtype(f.column()), - vocab.unintern_c( - vocab.get_interned(arg.string()) - ) - ); -#ifdef PSP_SSO_SCALAR - } else { - - a = coerce_to( - schema->get_dtype(f.column()), - arg.string().c_str() - ); - } -#endif - args.push_back(a); - break; - } - case proto::Scalar::kNull: - a.set(t_none()); - args.push_back(a); - break; - case proto::Scalar::SCALAR_NOT_SET: - PSP_COMPLAIN_AND_ABORT( - "Filter scalar type not implemented: " - + std::to_string(arg.scalar_case()) - ) - break; - } - } - - filter.emplace_back(f.column(), f.op(), args); - } - - const auto& cols = cfg.columns(); - std::vector columns; - if (cols.has_columns()) { - columns = { - cols.columns().columns().begin(), - cols.columns().columns().end() - }; - } else { - columns = table->get_column_names(); - for (const auto& f : expressions) { - columns.push_back(f->get_expression_alias()); - } - for (const auto& w : windows) { - columns.push_back(w.m_name); - } - } - - LOG_DEBUG( - "Creating view config with \n" - << "row_pivots: " << row_pivots << '\n' - << "column_pivots: " << column_pivots - << '\n' - // << "aggregates: " << aggregates << '\n' - << "columns: " << columns - << '\n' - // << "filter: " << filter << '\n' - << "sort_str: " << sort_str << '\n' - << "expressions: " << expressions << '\n' - << "column_only: " << column_only << '\n' - ); - - std::string filter_op; - switch (cfg.filter_op()) { - case proto::ViewConfig_FilterReducer:: - ViewConfig_FilterReducer_OR: - filter_op = "or"; - break; - case proto::ViewConfig_FilterReducer:: - ViewConfig_FilterReducer_AND: - default: - filter_op = "and"; - break; - } - - LOG_DEBUG("FILTER_OP: " << filter_op); - - bool leaves_only = - cfg.has_group_rollup_mode() ? cfg.group_rollup_mode() == 1 : false; - bool total_only = - cfg.has_group_rollup_mode() ? cfg.group_rollup_mode() == 2 : false; - bool split_rollup = cfg.has_split_rollup_mode() - && cfg.split_rollup_mode() - == proto::SplitRollupMode::SPLIT_ROLLUP_MODE_ROLLUP; - - auto config = std::make_shared( - vocab, - row_pivots, - column_pivots, - aggregates, - columns, - filter, - sort_str, - expressions, - filter_op, - column_only, - leaves_only, - total_only, - windows, - split_rollup - ); - config->init(schema); - - if (cfg.has_group_by_depth()) { - config->set_row_pivot_depth(cfg.group_by_depth()); - } - - std::uint32_t sides; - - if (!group_by.empty() || !split_by.empty()) { - if (!split_by.empty()) { - sides = 2; - } else { - sides = 1; - } - } else if (total_only) { - sides = 1; - } else { - sides = 0; - } - - bool is_unit_context = table->get_index().empty() && sides == 0 - && row_pivots.empty() && column_pivots.empty() - && aggregates.empty() && columns.empty() && sort_str.empty() - && cfg.expressions().empty() && cfg.windows().empty(); + auto built = build_view_config(table, r.config()); + const auto& schema = built.schema; + const auto& config = built.config; + const std::uint32_t sides = built.sides; + const bool is_unit_context = built.is_unit_context; std::shared_ptr erased_view; diff --git a/rust/perspective-server/cpp/perspective/src/cpp/view.cpp b/rust/perspective-server/cpp/perspective/src/cpp/view.cpp index c67e0815d7..24c20f4b94 100644 --- a/rust/perspective-server/cpp/perspective/src/cpp/view.cpp +++ b/rust/perspective-server/cpp/perspective/src/cpp/view.cpp @@ -439,87 +439,88 @@ View::column_paths_string() const { return out; } -template +static std::string +map_aggregate_type(t_aggtype agg, const std::string& typestring) { + switch (agg) { + case AGGTYPE_DISTINCT_COUNT: + case AGGTYPE_COUNT: { + return "integer"; + } break; + case AGGTYPE_MEAN: + case AGGTYPE_MEAN_BY_COUNT: + case AGGTYPE_WEIGHTED_MEAN: + case AGGTYPE_PCT_SUM_PARENT: + case AGGTYPE_PCT_SUM_GRAND_TOTAL: + case AGGTYPE_VARIANCE: + case AGGTYPE_STANDARD_DEVIATION: { + return "float"; + } break; + default: { + return typestring; + } break; + } +} + std::map -View::schema() const { - // TODO: should revert to m_table - auto schema = m_ctx->get_schema(); - auto _types = schema.types(); - auto names = schema.columns(); +describe_view_schema( + const t_view_config& config, const t_schema& schema, bool pivoted +) { + std::map out; + const auto type_of = [&schema](const std::string& name) { + return dtype_to_str( + schema.has_column(name) ? schema.get_dtype(name) : DTYPE_NONE + ); + }; - std::map types; - std::map new_schema; + if (!pivoted) { + for (const auto& name : config.get_columns()) { + if (name == "psp_okey") { + continue; + } - for (std::size_t i = 0, max = names.size(); i != max; ++i) { - types[names[i]] = _types[i]; + out[name] = type_of(name); + } + + return out; } - auto col_names = column_names(false); - for (const std::vector& name : col_names) { - // Pull out the main aggregate column - std::string agg_name = name.back().to_string(); - std::string type_string = dtype_to_str(types[agg_name]); - new_schema[agg_name] = type_string; + const bool map_types = + (!config.get_row_pivots().empty() || config.is_total_only()) + && (!config.is_column_only() || config.is_total_only()); - if ((!m_row_pivots.empty() || m_view_config->is_total_only()) && (!is_column_only() || m_view_config->is_total_only())) { - new_schema[agg_name] = - _map_aggregate_types(agg_name, new_schema[agg_name]); + for (const t_aggspec& agg : config.get_aggspecs()) { + const std::string& name = agg.name(); + if (name == "psp_okey") { + continue; + } + + std::string type_str = type_of(name); + if (map_types) { + type_str = map_aggregate_type(agg.agg(), type_str); } + + out[name] = type_str; } - return new_schema; + return out; +} + +template +std::map +View::schema() const { + return describe_view_schema(*m_view_config, m_ctx->get_schema(), true); } template <> std::map View::schema() const { - t_schema schema = m_ctx->get_schema(); - std::vector _types = schema.types(); - std::vector names = schema.columns(); - - std::map types; - for (std::size_t i = 0, max = names.size(); i != max; ++i) { - types[names[i]] = _types[i]; - } - - std::vector> cols = column_names(false); - std::map new_schema; - - for (auto& col : cols) { - std::string name = col.back().to_string(); - if (name == "psp_okey") { - continue; - } - new_schema[name] = dtype_to_str(types[name]); - } - - return new_schema; + return describe_view_schema(*m_view_config, m_ctx->get_schema(), false); } template <> std::map View::schema() const { - const t_schema& schema = m_ctx->get_schema(); - const std::vector& _types = schema.types(); - const std::vector& names = schema.columns(); - - std::map types; - for (std::size_t i = 0, max = names.size(); i != max; ++i) { - types[names[i]] = _types[i]; - } - - std::vector> cols = column_names(false); - std::map new_schema; - - for (auto& col : cols) { - std::string name = col.back().to_string(); - if (name == "psp_okey") { - continue; - } - new_schema[name] = dtype_to_str(types[name]); - } - - return new_schema; + return describe_view_schema(*m_view_config, m_ctx->get_schema(), false); } template @@ -1757,27 +1758,9 @@ std::string View::_map_aggregate_types( const std::string& name, const std::string& typestring ) const { - for (const t_aggspec& agg : m_aggregates) { if (agg.name() == name) { - switch (agg.agg()) { - case AGGTYPE_DISTINCT_COUNT: - case AGGTYPE_COUNT: { - return "integer"; - } break; - case AGGTYPE_MEAN: - case AGGTYPE_MEAN_BY_COUNT: - case AGGTYPE_WEIGHTED_MEAN: - case AGGTYPE_PCT_SUM_PARENT: - case AGGTYPE_PCT_SUM_GRAND_TOTAL: - case AGGTYPE_VARIANCE: - case AGGTYPE_STANDARD_DEVIATION: { - return "float"; - } break; - default: { - return typestring; - } break; - } + return map_aggregate_type(agg.agg(), typestring); } } diff --git a/rust/perspective-server/cpp/perspective/src/include/perspective/server.h b/rust/perspective-server/cpp/perspective/src/include/perspective/server.h index 4f23ee2278..e714f2b2f5 100644 --- a/rust/perspective-server/cpp/perspective/src/include/perspective/server.h +++ b/rust/perspective-server/cpp/perspective/src/include/perspective/server.h @@ -689,6 +689,21 @@ namespace server { std::vector> _handle_request(std::uint32_t client_id, Request&& req); + /** + * @brief A parsed `proto::ViewConfig`, shared by `TableMakeViewReq` + * and `TableDescribeReq`. + */ + struct BuiltViewConfig { + std::shared_ptr schema; + std::shared_ptr config; + std::uint32_t sides; + bool is_unit_context; + }; + + BuiltViewConfig build_view_config( + const std::shared_ptr
& table, const proto::ViewConfig& cfg + ); + std::vector> _poll(); void _process_table( diff --git a/rust/perspective-server/cpp/perspective/src/include/perspective/view.h b/rust/perspective-server/cpp/perspective/src/include/perspective/view.h index 16f2e6d926..8da6f5f712 100644 --- a/rust/perspective-server/cpp/perspective/src/include/perspective/view.h +++ b/rust/perspective-server/cpp/perspective/src/include/perspective/view.h @@ -461,4 +461,12 @@ class PERSPECTIVE_EXPORT View { std::shared_ptr m_view_config; }; + +/** + * @brief The schema a `View` built from `config` over `schema` reports, + * derived statically from the config with no context constructed. + */ +PERSPECTIVE_EXPORT std::map describe_view_schema( + const t_view_config& config, const t_schema& schema, bool pivoted +); } // end namespace perspective \ No newline at end of file diff --git a/rust/perspective-viewer/src/rust/agent/preamble.md b/rust/perspective-viewer/src/rust/agent/preamble.md index 431c44d119..89343f3c28 100644 --- a/rust/perspective-viewer/src/rust/agent/preamble.md +++ b/rust/perspective-viewer/src/rust/agent/preamble.md @@ -11,6 +11,7 @@ Panels: the viewer is a dashboard that can hold multiple panels (independent sid Rules: - Column names in configs must match the schema exactly, including case. +- A `set_view_config` or `add_panel` call that returns an error changed NOTHING - the viewer is exactly as it was, so there is nothing to undo. - If a tool returns an error, correct your input and retry. If the same call fails twice with the same error, stop and report the problem instead. - You have a fixed budget of model requests per prompt - be economical with tool calls. - After acting, answer with a single short sentence describing what changed. If the user asks a question you can answer from tool results, answer it directly. diff --git a/rust/perspective-viewer/src/rust/agent/tools.rs b/rust/perspective-viewer/src/rust/agent/tools.rs index 704c43a616..7f1adbce9f 100644 --- a/rust/perspective-viewer/src/rust/agent/tools.rs +++ b/rust/perspective-viewer/src/rust/agent/tools.rs @@ -873,22 +873,8 @@ impl PerspectiveViewerElement { async fn tool_set_view_config(&self, ctx: &ToolCtx, args: Value) -> Result { let args: SetViewConfigArgs = serde_json::from_value(args)?; let target = args.panel.as_deref(); - let panel = self.agent_panel(target)?; - let snapshot = - get_viewer_config(&panel.session, &panel.renderer, &self.presentation).await?; - + self.agent_panel(target)?; if let Err(err) = self.agent_restore(target, &args.config).await { - let rolled = self - .agent_restore(target, &serde_json::to_value(&snapshot)?) - .await; - - let err = match rolled { - Ok(()) => err, - Err(rollback_err) => ToolError(format!( - "{err} (additionally, restoring the prior config failed: {rollback_err})" - )), - }; - return Err(with_docs_hint(ctx, err)); } diff --git a/rust/perspective-viewer/src/rust/components/column_settings_sidebar/style_tab.rs b/rust/perspective-viewer/src/rust/components/column_settings_sidebar/style_tab.rs index 2dd75e0d2c..dc56d948a8 100644 --- a/rust/perspective-viewer/src/rust/components/column_settings_sidebar/style_tab.rs +++ b/rust/perspective-viewer/src/rust/components/column_settings_sidebar/style_tab.rs @@ -471,6 +471,7 @@ fn render_leaf(spec: ControlSpec, keys: &[String], ctx: &FieldRenderCtx) -> Opti max, step, include, + .. } => { let current = raw_config .as_ref() diff --git a/rust/perspective-viewer/src/rust/components/debug_panel.rs b/rust/perspective-viewer/src/rust/components/debug_panel.rs index 874bd6b73b..84b01c1c82 100644 --- a/rust/perspective-viewer/src/rust/components/debug_panel.rs +++ b/rust/perspective-viewer/src/rust/components/debug_panel.rs @@ -22,7 +22,6 @@ use crate::components::code_editor::CodeEditor; use crate::config::*; use crate::js::{MimeType, copy_to_clipboard, paste_from_clipboard}; use crate::presentation::*; -use crate::queries::fetch_hosted_tables; use crate::renderer::*; use crate::session::*; use crate::utils::*; @@ -283,21 +282,6 @@ impl DebugPanelProps { }, }; - if let OptionalUpdate::Update(name) = &config.table { - let hosted = fetch_hosted_tables(&props.workspace).await; - if !hosted - .iter() - .any(|(_, names)| names.iter().any(|n| n == name)) - { - fail( - format!("Unknown table \"{name}\""), - locate_key(&source, "table"), - ); - - return Ok(()); - } - } - let active = props.workspace.active_renderer().as_ref() == Some(&props.renderer); let result = crate::tasks::restore_panel( &props.session, @@ -308,6 +292,7 @@ impl DebugPanelProps { config, crate::tasks::RestoreErrors::Suppress, MissingTable::Error, + None, ) .await; @@ -318,7 +303,16 @@ impl DebugPanelProps { modified.set(false); props.set_text(text); }, - Err(e) => fail(format!("{e}"), (0, 0)), + Err(e) => { + let message = format!("{e}"); + let position = if message.starts_with("Unknown table") { + locate_key(&source, "table") + } else { + (0, 0) + }; + + fail(message, position) + }, } Ok(()) diff --git a/rust/perspective-viewer/src/rust/components/plugin_tab.rs b/rust/perspective-viewer/src/rust/components/plugin_tab.rs index 694c7ea101..b7683638f8 100644 --- a/rust/perspective-viewer/src/rust/components/plugin_tab.rs +++ b/rust/perspective-viewer/src/rust/components/plugin_tab.rs @@ -42,10 +42,10 @@ pub struct PluginTabProps { /// Active plugin's `plugin_config` bucket — threaded as a value /// snapshot from `RendererProps`. Changes on every mutation path /// that fires `plugin_config_changed` (in-tab edit, - /// `restore_and_render` JSON paste, `reset_all` with `all=true`) + /// `restore` JSON paste, `reset_all` with `all=true`) /// AND on plugin switch (the active bucket is keyed by plugin /// name, so `to_props()` produces a fresh `Rc` after - /// `commit_plugin_idx`). PluginTab is a pure function of this + /// a restore or edit commit). PluginTab is a pure function of this /// prop — no `Renderer::get_plugin_config()` reads against the /// interior-mutable handle. pub plugin_config: PtrEqRc>, @@ -256,6 +256,7 @@ fn render_leaf( max, step, include, + .. } => { let current = raw_config.get(&key).and_then(|v| v.as_f64()); Some(html! { diff --git a/rust/perspective-viewer/src/rust/components/viewer.rs b/rust/perspective-viewer/src/rust/components/viewer.rs index 61e4aca7d2..adc13d5af0 100644 --- a/rust/perspective-viewer/src/rust/components/viewer.rs +++ b/rust/perspective-viewer/src/rust/components/viewer.rs @@ -151,7 +151,7 @@ impl Component for PerspectiveViewer { let elem = ctx.props().elem.clone(); let fonts = FontLoaderProps::new(&elem, ctx.link().callback(|()| PreloadFontsUpdate)); let empty_session = Session::new(); - let empty_renderer = Renderer::new(&elem); + let empty_renderer = Renderer::new(&elem, empty_session.cell()); let active_session = ctx .props() .workspace @@ -256,7 +256,8 @@ impl Component for PerspectiveViewer { .into_iter() .filter_map(|id| workspace.panel(&id).map(|p| (id, p))) { - stamp_global_overlay(workspace, &id, &panel.session); + let _ = id; + drop(broadcast_overlay(workspace, &panel)); tasks.push(reset_all( &panel.session, &panel.renderer, diff --git a/rust/perspective-viewer/src/rust/components/viewer/filters.rs b/rust/perspective-viewer/src/rust/components/viewer/filters.rs index 8147ccb7ef..b577f39be9 100644 --- a/rust/perspective-viewer/src/rust/components/viewer/filters.rs +++ b/rust/perspective-viewer/src/rust/components/viewer/filters.rs @@ -23,6 +23,7 @@ use yew::prelude::*; use super::PerspectiveViewer; use super::msg::MasterSelection; +use crate::session::OverlayClause; use crate::tasks::*; use crate::workspace::PanelId; @@ -107,7 +108,18 @@ impl PerspectiveViewer { }, }; - workspace.set_contribution(&id, filters); + let metadata = workspace.panel(&id).map(|p| p.session.metadata()); + let clauses = filters + .into_iter() + .map(|filter| OverlayClause { + column_type: metadata + .as_ref() + .and_then(|m| m.get_column_table_type(filter.column())), + filter, + }) + .collect(); + + workspace.set_contribution(&id, clauses); apply_global_filters(workspace); false } diff --git a/rust/perspective-viewer/src/rust/config/column_config_schema.rs b/rust/perspective-viewer/src/rust/config/column_config_schema.rs index 79827603f9..455b91fcb0 100644 --- a/rust/perspective-viewer/src/rust/config/column_config_schema.rs +++ b/rust/perspective-viewer/src/rust/config/column_config_schema.rs @@ -61,6 +61,14 @@ impl ColumnConfigSchema { } } +/// A per-column statistic a [`ControlSpec::Number`] may take its default from. +#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum NumberStat { + /// The largest absolute value in the column. + AbsMax, +} + /// Discriminated union of widget kinds the viewer can render. Composite /// variants wrap an existing rich Yew component and carry only the /// component's `*DefaultConfig`. Primitive variants render generic scalar @@ -126,6 +134,11 @@ pub enum ControlSpec { #[serde(default, skip_serializing_if = "Option::is_none")] step: Option, + + /// The column statistic the HOST substitutes for `default`, once it has + /// fetched it. + #[serde(default, skip_serializing_if = "Option::is_none")] + default_stat: Option, }, String { key: String, @@ -311,6 +324,31 @@ impl ColumnConfigSchema { self.canonicalize_defaults().group_format_controls() } + /// Replace the `default` of every [`ControlSpec::Number`] that declares a + /// `default_stat` with that statistic, where the host has it. + pub fn resolve_stat_defaults(mut self, abs_max: Option) -> Self { + fn walk(fields: &mut [ControlSpec], abs_max: Option) { + for spec in fields { + match spec { + ControlSpec::Group { fields, .. } => walk(fields, abs_max), + ControlSpec::Number { + default, + default_stat: Some(NumberStat::AbsMax), + .. + } => { + if let Some(abs_max) = abs_max { + *default = abs_max; + } + }, + _ => {}, + } + } + } + + walk(&mut self.fields, abs_max); + self + } + pub fn group_format_controls(mut self) -> Self { fn is_format(spec: &ControlSpec) -> bool { matches!( diff --git a/rust/perspective-viewer/src/rust/config/options.rs b/rust/perspective-viewer/src/rust/config/options.rs index 9dbb397184..34e539ec25 100644 --- a/rust/perspective-viewer/src/rust/config/options.rs +++ b/rust/perspective-viewer/src/rust/config/options.rs @@ -35,13 +35,9 @@ pub struct RestoreOptions { #[ts(optional)] pub panel: Option, - /// When `true`, a failed restore only REJECTS the returned `Promise` — - /// the error is not committed to the viewer's visible error state, and - /// the session remains usable for subsequent calls. For programmatic - /// callers (e.g. the LLM agent's `set_view_config` tool) for whom a - /// failed config patch is feedback rather than a user-facing fault. - /// The config may be partially applied on failure; restore a - /// known-good config to recover exactly. + /// When `true`, a config that was applied and then failed to render only + /// rejects the returned `Promise`, without raising the panel's visible + /// error state. #[ts(optional)] pub suppress_errors: Option, diff --git a/rust/perspective-viewer/src/rust/custom_elements/viewer.rs b/rust/perspective-viewer/src/rust/custom_elements/viewer.rs index 9282838eb2..ff218840f7 100644 --- a/rust/perspective-viewer/src/rust/custom_elements/viewer.rs +++ b/rust/perspective-viewer/src/rust/custom_elements/viewer.rs @@ -19,7 +19,6 @@ use futures::channel::oneshot::channel; use futures::future::join_all; use js_sys::{Array, JsString}; use perspective_client::config::ViewConfigUpdate; -use perspective_client::utils::PerspectiveResultExt; use perspective_js::utils::global; use perspective_js::{JsViewConfig, JsViewWindow, Table, View, apierror}; use wasm_bindgen::JsCast; @@ -38,7 +37,9 @@ use crate::js::*; use crate::presentation::*; use crate::queries::*; use crate::root::Root; -use crate::session::{Disposal, MissingTable, ResetOptions, TableLoadState}; +use crate::session::{ + BindPlan, Disposal, MissingTable, OpKind, ResetOptions, StepOutcome, TableLoadState, +}; use crate::tasks::*; use crate::utils::*; use crate::workspace::{Panel, PanelId, Workspace}; @@ -212,13 +213,13 @@ impl PerspectiveViewerElement { .panels() .into_iter() .filter(|panel| !fresh.contains(&panel.id)) - .filter(|panel| stamp_global_overlay(&self.workspace, &panel.id, &panel.session)) .collect::>(); - join_all(retained.iter().map(|panel| async move { - apply_and_render(&panel.session, &panel.renderer, ViewConfigUpdate::default())?.await?; - Ok(()) - })) + join_all( + retained + .iter() + .map(|panel| broadcast_overlay(&self.workspace, panel)), + ) .await .into_iter() .collect::>>()?; @@ -548,23 +549,28 @@ impl PerspectiveViewerElement { let session = panel.session; let renderer = panel.renderer; - let load = session.begin_pending_load(); clone!(self.workspace, self.presentation); - Ok(ApiFuture::new_throttled(async move { - let _effect = effect; - enum Discard { - Unclaimed(Panel), - Evicted(Panel), - } + enum Discard { + Unclaimed(Panel), + Evicted(Panel), + } - renderer.set_throttle(None); - let _run_token = session.begin_config_run(); - let result = { - clone!(session, renderer, workspace, notify, load); - renderer + let table_known = matches!( + try_from_js_option::(table.clone()), + Ok(Some(_)) + ); + + let ticket = session.submit(OpKind::Load { table_known }, { + clone!(session, renderer); + move |load| { + Box::pin(async move { + let _effect = effect; + renderer.set_throttle(None); + let result = { + clone!(session, renderer, workspace, notify, load); + renderer .clone() .render_task(|guard| async move { - seed_panel_theme(&presentation, &renderer).await; renderer.stamp_theme(None); let jstable = JsFuture::from(promise) .await @@ -574,30 +580,17 @@ impl PerspectiveViewerElement { try_from_js_option::(jstable.clone()) { tracing::warn!("{}", DEPRECATED_TABLE_MESSAGE); - let Some(journal) = load.claim() else { + if load.is_superseded() { return Ok(None); - }; + } if let Some(notify) = ¬ify { place_reserved(&workspace, notify, true); } - let _plugin = renderer.ensure_plugin_selected()?; - let _ = renderer.mount_active_plugin(); - session - .reset(ResetOptions { - config: true, - expressions: true, - stats: true, - table: Some(session::TableIntermediateState::Reloaded), - }) - .await - .unwrap_or_log(); - let client = table.get_client().await; let inner_client = client.get_client().clone(); - session.set_client(inner_client.clone()); - workspace.set_default_client(inner_client); + workspace.set_default_client(inner_client.clone()); let name = table.get_name().await; tracing::debug!( "Loading {:.0} rows from `Table` {}", @@ -605,21 +598,40 @@ impl PerspectiveViewerElement { name ); - session.set_table(name).await?; - for delta in journal { - session.commit_view_config(delta)?; - } - - session.commit_table_defaults(); - let (disposition, _pin) = - crate::tasks::bind_snapshot(&guard, &session, &renderer).await?; + let plan = match session::probe_table( + &inner_client, + &name, + MissingTable::Pend, + ) + .await? + { + Some(table) => BindPlan::Bind { + client: inner_client, + table: Box::new(table), + reset: true, + }, + None => BindPlan::Pend { + client: inner_client, + name, + reset: true, + }, + }; - crate::tasks::dispatch_bound( - &guard, + let prepared = crate::tasks::prepare( + &session, &renderer, - disposition, + &presentation, false, - crate::tasks::RunOrigin::Public, + plan, + renderer + .slot_name() + .map(|id| overlay_for(&workspace, &PanelId::from(id))), + ViewerConfigUpdate::default(), + ) + .await?; + + crate::tasks::commit_and_render_locked( + guard, &session, &renderer, prepared, ) .await?; @@ -631,11 +643,8 @@ impl PerspectiveViewerElement { // INERT: register the client only — never rebind or // reset the active panel (its table is preserved). // Panels bind their client lazily at table-resolution - // time (`Workspace::resolve_client_for_table`). The - // window is discarded (not replayed): a `Client` - // performs no reset, and any racing `restore`'s - // commits already applied live (`commit_view_config`). - let owned_window = load.claim().is_some(); + // time (`Workspace::resolve_client_for_table`). + let owned_window = !load.is_superseded(); let discard = if owned_window && notify.is_some() { match workspace.take_reserved() { Some(panel) => Some(Discard::Unclaimed(panel)), @@ -671,34 +680,36 @@ impl PerspectiveViewerElement { workspace.set_default_client(client.get_client().clone()); Ok(discard) } else { - load.close(); Err(ApiError::new("Invalid argument")) } }) .await - }; + }; - match result { - Err(e) => { - load.close(); - if let Some(notify) = ¬ify { - place_reserved(&workspace, notify, true); - } + match result { + Err(e) => { + if let Some(notify) = ¬ify { + place_reserved(&workspace, notify, true); + } - session.set_error(false, e.clone()).await?; - Err(e) - }, - Ok(Some(Discard::Unclaimed(panel))) => { - eject_panel(panel, Disposal::Reject).await?; - Ok(()) - }, - Ok(Some(Discard::Evicted(panel))) => { - eject_panel(panel, Disposal::Resolve).await?; - Err(ApiError::new(CREATE_REQUIRES_TABLE)) - }, - Ok(None) => Ok(()), + session.set_error(false, e.clone()).await?; + Err(e) + }, + Ok(Some(Discard::Unclaimed(panel))) => { + eject_panel(panel, Disposal::Reject).await?; + Ok(StepOutcome::Done) + }, + Ok(Some(Discard::Evicted(panel))) => { + eject_panel(panel, Disposal::Resolve).await?; + Err(ApiError::new(CREATE_REQUIRES_TABLE)) + }, + Ok(None) => Ok(StepOutcome::Done), + } + }) } - })) + }); + + Ok(ApiFuture::new_throttled(ticket.settle())) } /// Delete all internal [`View`]s and all associated state, rendering this @@ -1006,6 +1017,7 @@ impl PerspectiveViewerElement { let mut fulfilled = false; for panel in &panels { + panel.session.settle_ops().await; panel.renderer.clone().with_lock(async { Ok(()) }).await?; panel.renderer.clone().with_lock(async { Ok(()) }).await?; panel.session.settle_dispatches().await?; @@ -1064,11 +1076,9 @@ impl PerspectiveViewerElement { /// either "json", "string" or "arraybuffer" format. /// - `options.panel` - The panel to target, or the active panel when /// omitted. - /// - `options.suppress_errors` - when `true`, a failed restore only rejects - /// the returned `Promise` and rolls the view config back, leaving the - /// viewer's visible error state and any element-level state the call - /// already applied (theme, title, plugin, a successful `table` change) - /// untouched. + /// - `options.suppress_errors` - when `true`, a config that was applied and + /// then failed to render only rejects the returned `Promise`, without + /// raising the panel's visible error state. /// - `options.wait_for_table` - when `true`, a `table` no loaded client /// hosts yet leaves the panel empty and pending until the table is /// created, instead of the default error. @@ -1142,14 +1152,24 @@ impl PerspectiveViewerElement { }, }; - if !matches!(settings, OptionalUpdate::Missing) { - let (sender, receiver) = channel::>(); - this.root.borrow().as_ref().into_apierror()?.send_message( - PerspectiveViewerMsg::ToggleSettingsInit(Some(settings), false, Some(sender)), - ); + let settings_toggle: Option>> = + (!matches!(settings, OptionalUpdate::Missing)).then(|| { + let this = this.clone(); + Box::pin(async move { + let (sender, receiver) = channel::>(); + this.root.borrow().as_ref().into_apierror()?.send_message( + PerspectiveViewerMsg::ToggleSettingsInit( + Some(settings), + false, + Some(sender), + ), + ); - receiver.await.map_err(|_| ApiError::new("Cancelled"))??; - } + receiver.await.map_err(|_| ApiError::new("Cancelled"))??; + Ok(()) + }) + as futures::future::LocalBoxFuture<'static, ApiResult<()>> + }); match target { Target::Existing { panel, active } => { @@ -1162,6 +1182,7 @@ impl PerspectiveViewerElement { update, errors, missing, + settings_toggle, ) .await }, @@ -1175,10 +1196,15 @@ impl PerspectiveViewerElement { update, errors, missing, + settings_toggle, ) .await }, Target::Create(config) => { + if let Some(settings_toggle) = settings_toggle { + settings_toggle.await?; + } + create_panel( &this.elem, &this.presentation, @@ -1242,9 +1268,9 @@ impl PerspectiveViewerElement { let results = join_all(contents.into_iter().map(|(id, session, renderer, config)| { let presentation = this.presentation.clone(); let workspace = this.workspace.clone(); + let notify = this.layout_changed_notify(); async move { - stamp_global_overlay(&workspace, &id, &session); - restore_panel( + let result = restore_panel( &session, &renderer, &presentation, @@ -1253,8 +1279,15 @@ impl PerspectiveViewerElement { config, crate::tasks::RestoreErrors::Publish, missing, + None, ) - .await?; + .await; + + if discard_rejected(&workspace, &id, &session, &result).await? { + notify.emit(()); + } + + result?; if workspace.is_master(&id) { set_edit_mode(&session, &renderer, "SELECT_ROW_TREE"); } @@ -1286,10 +1319,19 @@ impl PerspectiveViewerElement { }; let reset_effect = self.workspace.effects().guard(); - let reset_task = panel.session.reset(ResetOptions::default()); + let reset = panel.session.submit(OpKind::Restore { fields: None }, { + let session = panel.session.clone(); + move |_ctx| { + Box::pin(async move { + session.reset(ResetOptions::default()).await?; + Ok(StepOutcome::Done) + }) + } + }); + ApiFuture::spawn(async move { let _effect = reset_effect; - reset_task.await + reset.settle().await }); let effect = self.workspace.effects().guard(); diff --git a/rust/perspective-viewer/src/rust/js/plugin.rs b/rust/perspective-viewer/src/rust/js/plugin.rs index dccb0310e3..94d3712114 100644 --- a/rust/perspective-viewer/src/rust/js/plugin.rs +++ b/rust/perspective-viewer/src/rust/js/plugin.rs @@ -51,7 +51,7 @@ extern "C" { /// call — the view re-renders and re-queries the schema once the /// async fetch resolves. #[wasm_bindgen(method, catch, js_name = column_config_schema)] - pub fn _column_config_schema(this: &JsPerspectiveViewerPlugin, view_type: &str, group: Option<&str>, column_name: &str, current_value: &JsValue, view_config: &JsValue, column_stats: &JsValue) -> ApiResult; + pub fn _column_config_schema(this: &JsPerspectiveViewerPlugin, view_type: &str, group: Option<&str>, column_name: &str, current_value: &JsValue, view_config: &JsValue, plugin_config: &JsValue) -> ApiResult; /// The plugin-level settings schema as it applies to `current_value`, the /// `plugin_config` state the schema should describe. diff --git a/rust/perspective-viewer/src/rust/queries/plugin_column_styles.rs b/rust/perspective-viewer/src/rust/queries/plugin_column_styles.rs index a4c2365a72..697ed0449b 100644 --- a/rust/perspective-viewer/src/rust/queries/plugin_column_styles.rs +++ b/rust/perspective-viewer/src/rust/queries/plugin_column_styles.rs @@ -13,22 +13,11 @@ use itertools::Itertools; use perspective_client::config::ViewConfig; use perspective_js::utils::{ApiResult, JsValueSerdeExt}; -use serde::Serialize; use crate::config::ColumnConfigSchema; use crate::renderer::Renderer; use crate::session::SessionMetadata; -/// Stats payload passed to `plugin.column_config_schema` as the -/// `column_stats` arg. The caller (e.g. the StyleTab) owns the value — -/// typically populated by `fetch_column_abs_max` resolving into a -/// component-local `use_state`. Missing when no fetch has resolved yet. -#[derive(Default, Serialize)] -struct ColumnStats { - #[serde(skip_serializing_if = "Option::is_none")] - abs_max: Option, -} - /// Queries the active plugin for its plugin-scoped [`ColumnConfigSchema`] as /// it applies to `current_value`. pub fn get_plugin_config_schema( @@ -51,10 +40,6 @@ pub fn get_plugin_config_schema( /// /// `current_value` is the column's existing flat JSON config (if any); /// plugins use it to dynamically gate fields based on prior state. -/// `abs_max` is the caller-owned numeric stat (typically a Yew -/// `use_state` populated by an in-flight `fetch_column_abs_max` task); -/// `None` means the fetch has not yet resolved and gradient defaults -/// fall back to 0. pub fn get_column_config_schema( renderer: &Renderer, view_config: &ViewConfig, @@ -87,9 +72,8 @@ pub fn get_column_config_schema( let view_config_js = wasm_bindgen::JsValue::from_serde_ext(view_config).unwrap_or(wasm_bindgen::JsValue::NULL); - let stats = ColumnStats { abs_max }; - let stats_js = - wasm_bindgen::JsValue::from_serde_ext(&stats).unwrap_or(wasm_bindgen::JsValue::NULL); + let plugin_config_js = wasm_bindgen::JsValue::from_serde_ext(&renderer.get_plugin_config()) + .unwrap_or(wasm_bindgen::JsValue::NULL); let raw = plugin._column_config_schema( &view_type.to_string(), @@ -97,10 +81,10 @@ pub fn get_column_config_schema( column_name, ¤t_js, &view_config_js, - &stats_js, + &plugin_config_js, )?; serde_wasm_bindgen::from_value::(raw) - .map(|schema| schema.canonicalize()) + .map(|schema| schema.canonicalize().resolve_stat_defaults(abs_max)) .map_err(|e| e.into()) } diff --git a/rust/perspective-viewer/src/rust/renderer.rs b/rust/perspective-viewer/src/rust/renderer.rs index 04242d3813..34a8e5e19e 100644 --- a/rust/perspective-viewer/src/rust/renderer.rs +++ b/rust/perspective-viewer/src/rust/renderer.rs @@ -40,13 +40,17 @@ use yew::prelude::*; use self::activate::*; pub use self::limits::RenderLimits; -pub use self::plugin_config::{ColumnConfigMap, PluginScopedConfig}; +pub use self::plugin_config::{ + ColumnConfigMap, PluginScopedConfig, ValidatedColumnsConfig, ValidatedPluginConfig, + apply_columns_config_to, apply_plugin_config_to, +}; use self::plugin_store::*; pub use self::props::RendererProps; pub use self::registry::*; use self::render_timer::*; use crate::config::*; use crate::js::plugin::*; +use crate::session::{PanelCell, PluginRef}; use crate::utils::*; /// Minimum geometry delta (px) considered a real size/position change by the @@ -97,6 +101,8 @@ impl Drop for ContextPin { /// Immutable state pub struct RendererData { + /// This panel's committed [`PanelState`]. + cell: PanelCell, plugin_data: RefCell, draw_lock: DebounceMutex, pub plugin_changed: PubSub, @@ -151,20 +157,12 @@ pub struct RendererData { /// coexist there. slot_name: RefCell>, - /// This panel's theme name — CONCRETE, resolved once at creation from - /// the config's `theme`, else the host's, else the registry default. - /// There is no "unthemed panel" that resolves against live element - /// state at draw time: a registry re-ordering must never repaint a - /// panel, so nothing but an explicit write may change this. `None` only - /// while no themes exist at all. - theme: RefCell>, - /// Whether the active plugin has completed a draw. An EXPLICIT flag — /// not inferred from DOM connectedness — because plugin elements may be /// mounted eagerly (at panel creation / draw start, before the view /// query resolves), so "in the DOM" no longer implies "has rendered". /// Set by a successful `draw_view`; cleared on plugin swap - /// (`commit_plugin_idx`), `dispose` and `delete`. + /// (`activate_committed_plugin`), `dispose` and `delete`. has_drawn: Cell, /// The effective theme stamped at the active plugin's last `--psp-*` @@ -210,12 +208,9 @@ pub struct RendererData { /// Mutable state pub struct RendererMutData { viewer_elem: HtmlElement, - metadata: Rc, plugin_store: PluginStore, - plugins_idx: Option, timer: MovingWindowRenderTimer, selection: Option, - plugin_states: HashMap, } /// The state object responsible for the active [`JsPerspectiveViewerPlugin`]. @@ -247,17 +242,15 @@ impl Deref for RendererData { } impl Renderer { - pub fn new(viewer_elem: &HtmlElement) -> Self { + pub fn new(viewer_elem: &HtmlElement, cell: PanelCell) -> Self { let draw_lock = DebounceMutex::default(); Self(Rc::new(RendererData { + cell, plugin_data: RefCell::new(RendererMutData { viewer_elem: viewer_elem.clone(), - metadata: Rc::new(PluginStaticConfig::default()), plugin_store: PluginStore::default(), - plugins_idx: None, selection: None, timer: MovingWindowRenderTimer::default(), - plugin_states: HashMap::default(), }), geometry_slot: draw_lock.slot(), geometry_cmd: Cell::new(None), @@ -274,7 +267,6 @@ impl Renderer { presized_box: Cell::new(None), on_render_limits_changed: Default::default(), slot_name: Default::default(), - theme: Default::default(), has_drawn: Cell::new(false), captured_theme: Default::default(), cached_context: Default::default(), @@ -301,12 +293,24 @@ impl Renderer { /// resolve "the default" through `Presentation` before calling, never by /// leaving this empty. pub fn set_theme(&self, name: Option) { - *self.0.theme.borrow_mut() = name; + self.cell.submit_theme(name); + } + + /// Write this panel's theme name NOW. + pub fn commit_theme(&self, name: Option) { + self.cell.swap(self.cell.state().with_theme(name)); + } + + /// [`Self::commit_theme`] plus a synchronous [`Self::stamp_theme`]. + pub fn commit_theme_stamped(&self, theme: Option) { + self.commit_theme(theme); + self.stamp_theme(None); } - /// This panel's theme name. + /// This panel's theme name, as the UI sees it: the committed theme, or the + /// latest pending pick. pub fn theme(&self) -> Option { - self.0.theme.borrow().clone() + self.cell.projected_theme() } /// [`Self::set_theme`] plus a synchronous [`Self::stamp_theme`] — the @@ -376,17 +380,25 @@ impl Renderer { plugin.delete(); } self.plugin_data.borrow().viewer_elem.set_inner_text(""); - let new_state = Self::new(&self.plugin_data.borrow().viewer_elem); - std::mem::swap( - &mut *self.plugin_data.borrow_mut(), - &mut *new_state.plugin_data.borrow_mut(), - ); + let viewer_elem = self.plugin_data.borrow().viewer_elem.clone(); + *self.plugin_data.borrow_mut() = RendererMutData { + viewer_elem, + plugin_store: PluginStore::default(), + selection: None, + timer: MovingWindowRenderTimer::default(), + }; + self.cell.swap(self.cell.state().without_plugins()); Ok(()) } pub fn metadata(&self) -> Rc { - self.borrow().metadata.clone() + self.cell + .state() + .plugin + .as_ref() + .map(|plugin| plugin.static_config.clone()) + .unwrap_or_default() } pub fn is_chart(&self) -> bool { @@ -440,7 +452,7 @@ impl Renderer { pub fn active_plugin(&self) -> Option { // Bail on `plugins_idx` BEFORE touching `plugin_store`, so an unselected // renderer never snapshots the registry. - let idx = self.0.borrow().plugins_idx?; + let idx = self.selected_idx()?; self.0.borrow_mut().plugin_store.plugins().get(idx).cloned() } @@ -556,7 +568,7 @@ impl Renderer { let idx = self.find_plugin_idx(name)?; let changed = !matches!( - self.0.borrow().plugins_idx, + self.selected_idx(), Some(selected_idx) if selected_idx == idx ); @@ -574,6 +586,24 @@ impl Renderer { } } + /// Reject a [`PluginUpdate`] naming a plugin that is not registered. + pub fn check_plugin_update(&self, update: &PluginUpdate) -> ApiResult<()> { + if let PluginUpdate::Update(name) = update + && self.find_plugin_idx(name).is_none() + { + let known = self + .get_all_plugin_configs() + .iter() + .map(|c| format!("\"{}\"", c.name)) + .collect::>() + .join(", "); + + return Err(format!("Unknown plugin \"{name}\"; expected one of {known}").into()); + } + + Ok(()) + } + /// Commit a plugin selection previously resolved by /// [`Self::resolve_plugin_update`]. COMMAND — call only from inside a /// locked draw task, so the swap lands atomically with the view rebuild @@ -586,7 +616,7 @@ impl Renderer { let idx = match idx { Some(idx) => idx, None => { - if self.0.borrow().plugins_idx.is_none() { + if self.selected_idx().is_none() { let name = PLUGIN_REGISTRY.default_plugin_name(); let idx = self .find_plugin_idx(&name) @@ -600,7 +630,7 @@ impl Renderer { }; let changed = !matches!( - self.0.borrow().plugins_idx, + self.selected_idx(), Some(selected_idx) if selected_idx == idx ); @@ -616,12 +646,6 @@ impl Renderer { /// reset the per-plugin render-warning flag, and fire /// `plugin_changed`. fn commit_plugin_idx(&self, idx: usize) -> ApiResult<()> { - // The newly-selected plugin element has not drawn (a swap keeps the - // OLD plugin mounted until the new one's draw lands) — and has - // captured no CSS. - self.0.has_drawn.set(false); - self.0.captured_theme.borrow_mut().take(); - self.borrow_mut().plugins_idx = Some(idx); let config = self .0 .borrow_mut() @@ -631,22 +655,46 @@ impl Renderer { .cloned() .ok_or("No Plugin")?; - self.borrow_mut().metadata = config.clone(); + self.cell.swap(self.cell.state().with_plugin(PluginRef { + idx, + static_config: config, + })); + + self.activate_committed_plugin() + } + + /// The `static_config` of the plugin at store index `idx`, for a + /// [`PluginRef`] committed by a transaction rather than by + /// [`Self::commit_plugin`]. + pub fn plugin_ref(&self, idx: usize) -> ApiResult { + let static_config = self + .0 + .borrow_mut() + .plugin_store + .plugin_configs() + .get(idx) + .cloned() + .ok_or("No Plugin")?; + + Ok(PluginRef { idx, static_config }) + } + + /// Bring the plugin ELEMENT in line with a just-committed selection: the + /// newly-selected element has drawn nothing and captured no CSS, takes its + /// own stored bucket, and is announced. + pub fn activate_committed_plugin(&self) -> ApiResult<()> { + // The newly-selected plugin element has not drawn (a swap keeps the + // OLD plugin mounted until the new one's draw lands) — and has + // captured no CSS. + self.0.has_drawn.set(false); + self.0.captured_theme.borrow_mut().take(); self.0.render_warning.set(true); - // `commit_plugin_idx` is called *by* the selection path, so it must use - // the pure query (never `ensure_plugin_selected`, which would recurse - // through `commit_plugin`). `plugins_idx` was just set above. let plugin: JsPerspectiveViewerPlugin = self.active_plugin().ok_or("No Plugin")?; // Push the newly-activated plugin's stored bucket through // `plugin.restore` so the swap immediately reflects any // viewer-owned per-column and plugin-level config. - let bucket = self - .borrow() - .plugin_states - .get(&config.name) - .cloned() - .unwrap_or_default(); + let bucket = self.cell.state().bucket(&self.metadata().name); let token = JsValue::from_serde_ext(&bucket.plugin).unwrap_or(JsValue::NULL); if let Err(e) = plugin.restore(&token, Some(&bucket.columns)) { tracing::warn!("plugin.restore on swap failed: {:?}", e); @@ -656,6 +704,20 @@ impl Renderer { Ok(()) } + /// The committed plugin selection's index into the plugin store. + pub fn committed_plugin_idx(&self) -> Option { + self.selected_idx() + } + + /// The COMMITTED panel theme, pending theme edits excluded. + pub fn committed_theme(&self) -> Option { + self.cell.state().chrome.theme.clone() + } + + fn selected_idx(&self) -> Option { + self.cell.state().plugin.as_ref().map(|plugin| plugin.idx) + } + pub fn render_timer(&self) -> MovingWindowRenderTimer { self.0.borrow().timer.clone() } diff --git a/rust/perspective-viewer/src/rust/renderer/plugin_config.rs b/rust/perspective-viewer/src/rust/renderer/plugin_config.rs index cc988bb050..9abbfb5ca0 100644 --- a/rust/perspective-viewer/src/rust/renderer/plugin_config.rs +++ b/rust/perspective-viewer/src/rust/renderer/plugin_config.rs @@ -11,23 +11,48 @@ // ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ //! The [`Renderer`]'s per-plugin config state: the plugin-level and -//! per-column buckets (`plugin_states`), their schema-aware strip/merge +//! per-column buckets (`PanelState::buckets`), their schema-aware strip/merge //! write paths, and the restore-prep materialized snapshot. -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; +use std::rc::Rc; use futures::future::join_all; use perspective_client::config::ViewConfig; +use perspective_client::proto::ColumnType; use perspective_js::utils::{ApiError, ApiResult, JsValueSerdeExt}; use serde_json::Value; use wasm_bindgen::prelude::*; use super::Renderer; use crate::config::*; +use crate::js::plugin::JsPerspectiveViewerPlugin; use crate::queries::resolve_abs_max; use crate::session::Session; use crate::utils::{CssKind, CssLiteralUse, parse_var_ref, resolve_css_refs}; +type ConfigMap = serde_json::Map; + +/// A schema query's answer: `Ok(None)` when the plugin declares no schema. +pub type SchemaResult = Result, ValidationError>; + +/// Everything a column's schema is a function of, besides the column itself. +struct ColumnSchemaEnv<'a> { + view_config: &'a ViewConfig, + session: &'a Session, + + /// The view types of a config not yet bound to a `View`; `None` reads the + /// bound `View`'s schema. + view_schema: Option<&'a HashMap>, + + /// Whether the session's column stats describe this config's table (`false` + /// for a restore that replaces it). + column_stats: bool, + + /// The plugin-level config the column will live under. + plugin_config: &'a ConfigMap, +} + /// A per-column config map. Each inner [`serde_json::Map`] is a flat collection /// of plugin-defined JSON keys whose shape is dictated by the active plugin's /// [`crate::config::ColumnConfigSchema`]. @@ -35,7 +60,7 @@ pub type ColumnConfigMap = HashMap, } +/// The plugin a bucket update is validated against and written to — named +/// EXPLICITLY, so a validation that precedes a plugin swap can name the swap +/// target rather than whichever plugin happens to be active. +pub struct PluginTarget { + pub element: JsPerspectiveViewerPlugin, + pub static_config: Rc, +} + +/// What [`Renderer::apply_plugin_config`] writes for a validated +/// [`PluginConfigUpdate`] — every fallible check already done. +#[derive(Clone, Debug, PartialEq)] +pub enum ValidatedPluginConfig { + Missing, + Clear, + Set { + /// Keys to insert, already normalized and filtered to the schema's + /// active keys. + map: ConfigMap, + + /// Keys the update set to their declared default: removed from the + /// bucket rather than stored. + remove: Vec, + + /// The schema's active keys, when a schema was available; the bucket is + /// pruned to them after the write. + active: Option>, + }, +} + +/// What [`Renderer::apply_columns_config`] writes for a validated +/// [`ColumnConfigUpdate`]: per column, the normalized entry (an EMPTY entry +/// removes the column). +#[derive(Debug, PartialEq)] +pub enum ValidatedColumnsConfig { + Missing, + Clear, + Set(Vec<(String, ConfigMap)>), +} + +/// A bucket value the schema rejects. +#[derive(Debug, PartialEq)] +pub struct ValidationError(pub String); + +impl From for ApiError { + fn from(err: ValidationError) -> Self { + ApiError::from(JsValue::from_str(&err.0)) + } +} + +/// Validate a plugin-level config update against the schema the merged result +/// would have. +pub fn validate_plugin_config( + current: &ConfigMap, + update: PluginConfigUpdate, + schema_of: &dyn Fn(&ConfigMap) -> SchemaResult, +) -> Result { + match update { + OptionalUpdate::SetDefault => Ok(ValidatedPluginConfig::Clear), + OptionalUpdate::Missing => Ok(ValidatedPluginConfig::Missing), + OptionalUpdate::Update(mut map) => { + let mut merged = current.clone(); + for (k, v) in &map { + merged.insert(k.clone(), v.clone()); + } + + let schema = schema_of(&merged)?; + let mut active = None; + let mut remove = vec![]; + if let Some(s) = &schema { + let keys = s.active_keys(); + map.retain(|k, _| keys.contains(k)); + active = Some(keys); + let errors = normalize_values(s, &mut map); + if let Some((key, error)) = errors.first() { + return Err(ValidationError(format!( + "Invalid `plugin_config.{key}`: {error}" + ))); + } + + let leaves = s.leaf_fields(); + map.retain(|key, value| { + let is_default = leaves + .iter() + .any(|spec| matches_declared_default(spec, key, value)); + if is_default { + remove.push(key.clone()); + } + + !is_default + }); + } + + Ok(ValidatedPluginConfig::Set { + map, + remove, + active, + }) + }, + } +} + +/// Validate a per-column config update, column by column, against the schema +/// each column's entry would have. +pub fn validate_columns_config( + update: ColumnConfigUpdate, + schema_of: &dyn Fn(&str, &ConfigMap) -> SchemaResult, + view_type_of: &dyn Fn(&str) -> Option, + resolve_css: &dyn Fn(&str, &mut ConfigMap), +) -> Result { + match update { + OptionalUpdate::SetDefault => Ok(ValidatedColumnsConfig::Clear), + OptionalUpdate::Missing => Ok(ValidatedColumnsConfig::Missing), + OptionalUpdate::Update(map) => { + let mut stripped = Vec::with_capacity(map.len()); + for (col, mut cfg) in map { + if let Some(schema) = schema_of(&col, &cfg)? { + let active = schema.active_keys(); + cfg.retain(|k, _| active.contains(k)); + let errors = normalize_values(&schema, &mut cfg); + if let Some((key, error)) = errors.first() { + let ty = view_type_of(&col) + .map(|t| format!(" for a {t} column")) + .unwrap_or_default(); + + return Err(ValidationError(format!( + "Invalid `columns_config[\"{col}\"].{key}`{ty}: {error}" + ))); + } + + resolve_css(&col, &mut cfg); + strip_default_values(&schema, &mut cfg); + } + + stripped.push((col, cfg)); + } + + Ok(ValidatedColumnsConfig::Set(stripped)) + }, + } +} + +/// Whether `target`'s plugin element declares the schema method `name`. +fn declares(target: &PluginTarget, name: &str) -> bool { + js_sys::Reflect::get(&target.element, &JsValue::from_str(name)) + .map(|f| f.is_function()) + .unwrap_or(false) +} + +/// Write a validated plugin-level update into `bucket`. +pub fn apply_plugin_config_to( + bucket: &mut PluginScopedConfig, + validated: ValidatedPluginConfig, +) -> bool { + match validated { + ValidatedPluginConfig::Missing => false, + ValidatedPluginConfig::Clear => { + let changed = !bucket.plugin.is_empty(); + bucket.plugin.clear(); + changed + }, + ValidatedPluginConfig::Set { + map, + remove, + active, + } => { + let mut changed = false; + for key in remove { + if bucket.plugin.remove(&key).is_some() { + changed = true; + } + } + + for (k, v) in map { + let prev = bucket.plugin.insert(k, v.clone()); + if prev.as_ref() != Some(&v) { + changed = true; + } + } + + if let Some(active) = active { + let before = bucket.plugin.len(); + bucket.plugin.retain(|k, _| active.contains(k)); + changed |= bucket.plugin.len() != before; + } + + changed + }, + } +} + +/// Write a validated per-column update into `bucket`. +pub fn apply_columns_config_to( + bucket: &mut PluginScopedConfig, + validated: ValidatedColumnsConfig, +) -> bool { + match validated { + ValidatedColumnsConfig::Missing => false, + ValidatedColumnsConfig::Clear => { + let was_nonempty = !bucket.columns.is_empty(); + bucket.columns.clear(); + was_nonempty + }, + ValidatedColumnsConfig::Set(entries) => { + let mut changed = false; + for (col, cfg) in entries { + if cfg.is_empty() { + if bucket.columns.remove(&col).is_some() { + changed = true; + } + } else { + match bucket.columns.insert(col, cfg.clone()) { + None => changed = true, + Some(old) if old != cfg => changed = true, + _ => {}, + } + } + } + + changed + }, + } +} + impl Renderer { /// Name of the currently-active plugin (used as the key into - /// `plugin_states`). Returns `None` when no plugin has been + /// `PanelState::buckets`). Returns `None` when no plugin has been /// activated yet. fn active_plugin_name(&self) -> Option { - Some(self.borrow().metadata.name.clone()).filter(|n| !n.is_empty()) + Some(self.metadata().name.clone()).filter(|n| !n.is_empty()) } - /// Snapshot of the active plugin's per-column config map. + /// Rewrite the named plugin's bucket: `f` edits a COPY, which is then + /// swapped in as one new [`crate::session::PanelState`]. + fn edit_bucket(&self, name: &str, f: impl FnOnce(&mut PluginScopedConfig) -> R) -> R { + let state = self.cell.state(); + let mut bucket = state.bucket(name); + let result = f(&mut bucket); + self.cell.swap(state.with_bucket(name, bucket)); + result + } + + /// The active plugin as a [`PluginTarget`], selecting the registry default + /// when none is selected yet (the command half, like + /// [`Renderer::ensure_plugin_selected`]). + pub fn active_target(&self) -> ApiResult { + let element = self.ensure_plugin_selected()?; + Ok(PluginTarget { + element, + static_config: self.metadata(), + }) + } + + /// The plugin at registry index `idx` as a [`PluginTarget`], WITHOUT + /// selecting it — for validating against a swap target before the swap. + pub fn target_at(&self, idx: usize) -> ApiResult { + let mut st = self.borrow_mut(); + let element = st.plugin_store.plugins().get(idx).cloned(); + let static_config = st.plugin_store.plugin_configs().get(idx).cloned(); + Ok(PluginTarget { + element: element.ok_or("No Plugin")?, + static_config: static_config.ok_or("No Plugin")?, + }) + } + + /// Write a validated plugin-level update into `target`'s bucket. + pub fn apply_plugin_config( + &self, + target: &PluginTarget, + validated: ValidatedPluginConfig, + ) -> bool { + self.edit_bucket(&target.static_config.name, |bucket| { + apply_plugin_config_to(bucket, validated) + }) + } + + /// Write a validated per-column update into `target`'s bucket. + pub fn apply_columns_config( + &self, + target: &PluginTarget, + validated: ValidatedColumnsConfig, + ) -> bool { + self.edit_bucket(&target.static_config.name, |bucket| { + apply_columns_config_to(bucket, validated) + }) + } + + /// The plugin a [`PluginUpdate`] would END on, resolved WITHOUT selecting + /// anything: the swap target when the update names a different plugin (with + /// its store index, to commit), else the selected plugin, else the registry + /// default a first run would select. + pub fn resolve_target( + &self, + update: &PluginUpdate, + ) -> ApiResult<(PluginTarget, Option)> { + if let Some((idx, _)) = self.resolve_plugin_update(update) { + return Ok((self.target_at(idx)?, Some(idx))); + } + + if let Some(plugin) = self.cell.state().plugin.as_ref() { + return Ok((self.target_at(plugin.idx)?, None)); + } + + let (idx, _) = self + .resolve_plugin_update(&OptionalUpdate::SetDefault) + .ok_or("No Plugin")?; + + Ok((self.target_at(idx)?, Some(idx))) + } + + /// Validate BOTH bucket updates of a restore against `target` and the view + /// the restore would build (`view_config`, whose columns have the types in + /// `view_schema`) — before either is written, and without a `View`. + #[allow(clippy::too_many_arguments)] + pub fn prepare_buckets( + &self, + target: &PluginTarget, + session: &Session, + view_config: &ViewConfig, + view_schema: Option<&HashMap>, + rebinds: bool, + plugin_config: PluginConfigUpdate, + columns_config: ColumnConfigUpdate, + ) -> ApiResult<(ValidatedPluginConfig, ValidatedColumnsConfig)> { + let current = self.cell.state().bucket(&target.static_config.name); + let plugin = validate_plugin_config(¤t.plugin, plugin_config, &|merged| { + if !declares(target, "plugin_config_schema") { + return Ok(None); + } + + self.query_plugin_config_schema_for(target, view_config, Some(merged)) + .map(Some) + .map_err(|e| ValidationError(format!("`plugin_config` schema failed: {e}"))) + })?; + + let mut ends_with = current.clone(); + apply_plugin_config_to(&mut ends_with, plugin.clone()); + let columns = validate_columns_config( + columns_config, + &|col, cfg| { + let Some(view_schema) = view_schema else { + return Ok(None); + }; + + if !declares(target, "column_config_schema") { + return Ok(None); + } + + let env = ColumnSchemaEnv { + view_config, + session, + view_schema: Some(view_schema), + column_stats: !rebinds, + plugin_config: &ends_with.plugin, + }; + + self.query_column_config_schema_for(target, &env, col, Some(cfg)) + .map(Some) + .map_err(|e| { + ValidationError(format!("`columns_config[\"{col}\"]` schema failed: {e}")) + }) + }, + &|col| view_schema.and_then(|x| x.get(col).copied()), + &|col, cfg| self.resolve_css_refs_in(col, cfg), + )?; + + Ok((plugin, columns)) + } + + /// The active plugin's per-column config map as the UI and `save()` see it + /// — pending style edits included. pub fn all_columns_configs(&self) -> ColumnConfigMap { self.active_plugin_name() - .and_then(|n| { - self.borrow() - .plugin_states - .get(&n) - .map(|b| b.columns.clone()) - }) + .map(|n| self.cell.projected_bucket(&n).columns) + .unwrap_or_default() + } + + /// The active plugin's COMMITTED per-column config map — what a running + /// op's step reads, since the edits queued behind it have not happened. + pub fn committed_columns_configs(&self) -> ColumnConfigMap { + self.active_plugin_name() + .map(|n| self.cell.state().bucket(&n).columns) .unwrap_or_default() } @@ -119,7 +508,7 @@ impl Renderer { view_config: &ViewConfig, session: &Session, ) -> ColumnConfigMap { - let mut configs = self.all_columns_configs(); + let mut configs = self.committed_columns_configs(); let mut to_warm: Vec = vec![]; for (col, entry) in &configs { if session @@ -140,6 +529,7 @@ impl Renderer { ControlSpec::Number { key, include: Some(true), + default_stat: Some(_), .. } if !entry.contains_key(key) ) @@ -194,12 +584,7 @@ impl Renderer { /// Clear the active plugin's per-column config map. pub fn reset_columns_configs(&self) { if let Some(n) = self.active_plugin_name() { - self.borrow_mut() - .plugin_states - .entry(n) - .or_default() - .columns - .clear(); + self.edit_bucket(&n, |bucket| bucket.columns.clear()); } } @@ -210,83 +595,47 @@ impl Renderer { column_name: &str, ) -> Option> { let n = self.active_plugin_name()?; - self.borrow() - .plugin_states - .get(&n)? + self.cell + .projected_bucket(&n) .columns .get(column_name) .cloned() } - /// Wholesale update the active plugin's per-column config map. + /// Wholesale update the active plugin's per-column config map: + /// [`validate_columns_config`] against the active target, then + /// [`Renderer::apply_columns_config`]. pub fn update_columns_configs( &self, view_config: &ViewConfig, session: &Session, update: ColumnConfigUpdate, ) -> ApiResult { - let Some(n) = self.active_plugin_name() else { + if self.active_plugin_name().is_none() { return Ok(false); - }; - - match update { - OptionalUpdate::SetDefault => { - let mut st = self.borrow_mut(); - let bucket = st.plugin_states.entry(n).or_default(); - let was_nonempty = !bucket.columns.is_empty(); - bucket.columns.clear(); - Ok(was_nonempty) - }, - OptionalUpdate::Missing => Ok(false), - OptionalUpdate::Update(map) => { - let mut stripped: Vec<(String, serde_json::Map)> = - Vec::with_capacity(map.len()); - for (col, mut cfg) in map { - if let Ok(schema) = - self.query_column_config_schema(view_config, session, &col, Some(&cfg)) - { - let active = schema.active_keys(); - cfg.retain(|k, _| active.contains(k)); - let errors = normalize_values(&schema, &mut cfg); - if let Some((key, error)) = errors.first() { - let ty = session - .metadata() - .get_column_view_type(&col) - .map(|t| format!(" for a {t} column")) - .unwrap_or_default(); - - return Err(ApiError::from(JsValue::from_str(&format!( - "Invalid `columns_config[\"{col}\"].{key}`{ty}: {error}" - )))); - } - - self.resolve_css_refs_in(&col, &mut cfg); - strip_default_values(&schema, &mut cfg); - } - - stripped.push((col, cfg)); - } + } - let mut st = self.borrow_mut(); - let bucket = st.plugin_states.entry(n).or_default(); - let mut changed = false; - for (col, cfg) in stripped { - if cfg.is_empty() { - if bucket.columns.remove(&col).is_some() { - changed = true; - } - } else { - match bucket.columns.insert(col, cfg.clone()) { - None => changed = true, - Some(old) if old != cfg => changed = true, - _ => {}, - } - } - } + let target = self.active_target()?; + let validated = validate_columns_config( + update, + &|col, cfg| { + let env = ColumnSchemaEnv { + view_config, + session, + view_schema: None, + column_stats: true, + plugin_config: &self.committed_plugin_config(), + }; - Ok(changed) + Ok(self + .query_column_config_schema_for(&target, &env, col, Some(cfg)) + .ok()) }, - } + &|col| session.metadata().get_column_view_type(col), + &|col, cfg| self.resolve_css_refs_in(col, cfg), + )?; + + Ok(self.apply_columns_config(&target, validated)) } /// Apply a single schema-field update from the column-style UI to @@ -304,7 +653,14 @@ impl Renderer { return; }; - let current_value = self.get_columns_config(&column_name); + let current_value = self + .cell + .state() + .bucket(&n) + .columns + .get(&column_name) + .cloned(); + if let Ok(schema) = self.query_column_config_schema( view_config, session, @@ -319,9 +675,7 @@ impl Renderer { strip_default_values(&schema, &mut update.value); } - let next = { - let mut st = self.borrow_mut(); - let bucket = st.plugin_states.entry(n.clone()).or_default(); + let next = self.edit_bucket(&n, |bucket| { let entry = bucket.columns.entry(column_name.clone()).or_default(); for k in &update.keys { entry.remove(k); @@ -333,7 +687,7 @@ impl Renderer { } entry.clone() - }; + }); // The schema AFTER the update decides which keys survive and, for a // key whose control kind follows another key (`fg_color` reads as a @@ -345,44 +699,45 @@ impl Renderer { .query_column_config_schema(view_config, session, &column_name, Some(&next)) .ok(); - let mut st = self.borrow_mut(); - let bucket = st.plugin_states.entry(n).or_default(); - if let Some(entry) = bucket.columns.get_mut(&column_name) { - if let Some(schema) = &after { - let active = schema.active_keys(); - entry.retain(|k, _| active.contains(k)); - for (key, error) in normalize_values(schema, entry) { - tracing::warn!("Dropping `{column_name}`.`{key}` after a mode change: {error}"); + self.edit_bucket(&n, |bucket| { + if let Some(entry) = bucket.columns.get_mut(&column_name) { + if let Some(schema) = &after { + let active = schema.active_keys(); + entry.retain(|k, _| active.contains(k)); + for (key, error) in normalize_values(schema, entry) { + tracing::warn!( + "Dropping `{column_name}`.`{key}` after a mode change: {error}" + ); + } } - } - if entry.is_empty() { - bucket.columns.remove(&column_name); + if entry.is_empty() { + bucket.columns.remove(&column_name); + } } - } + }) } - /// Snapshot of the active plugin's plugin-level config map. + /// The active plugin's plugin-level config map as the UI and `save()` see + /// it — pending settings edits included. pub fn get_plugin_config(&self) -> serde_json::Map { self.active_plugin_name() - .and_then(|n| { - self.borrow() - .plugin_states - .get(&n) - .map(|b| b.plugin.clone()) - }) + .map(|n| self.cell.projected_bucket(&n).plugin) + .unwrap_or_default() + } + + /// The active plugin's COMMITTED plugin-level config map — what a running + /// op's step reads. + pub fn committed_plugin_config(&self) -> serde_json::Map { + self.active_plugin_name() + .map(|n| self.cell.state().bucket(&n).plugin) .unwrap_or_default() } /// Clear the active plugin's plugin-level config map. pub fn reset_plugin_config(&self) { if let Some(n) = self.active_plugin_name() { - self.borrow_mut() - .plugin_states - .entry(n) - .or_default() - .plugin - .clear(); + self.edit_bucket(&n, |bucket| bucket.plugin.clear()); } } @@ -392,12 +747,25 @@ impl Renderer { fn query_plugin_config_schema( &self, view_config: &ViewConfig, - current_value: Option<&serde_json::Map>, + current_value: Option<&ConfigMap>, + ) -> ApiResult { + let target = self.active_target()?; + self.query_plugin_config_schema_for(&target, view_config, current_value) + } + + /// [`Self::query_plugin_config_schema`] against an explicit target. + fn query_plugin_config_schema_for( + &self, + target: &PluginTarget, + view_config: &ViewConfig, + current_value: Option<&ConfigMap>, ) -> ApiResult { - let plugin = self.ensure_plugin_selected()?; let view_config_js = JsValue::from_serde_ext(view_config).unwrap_or(JsValue::NULL); let current_js = JsValue::from_serde_ext(¤t_value).unwrap_or(JsValue::NULL); - let raw = plugin._plugin_config_schema(&view_config_js, ¤t_js)?; + let raw = target + .element + ._plugin_config_schema(&view_config_js, ¤t_js)?; + serde_wasm_bindgen::from_value::(raw) .map(|schema| schema.canonicalize()) .map_err(|e| e.into()) @@ -411,11 +779,40 @@ impl Renderer { view_config: &ViewConfig, session: &Session, column_name: &str, - current_value: Option<&serde_json::Map>, + current_value: Option<&ConfigMap>, + ) -> ApiResult { + let target = self.active_target()?; + let env = ColumnSchemaEnv { + view_config, + session, + view_schema: None, + column_stats: true, + plugin_config: &self.committed_plugin_config(), + }; + + self.query_column_config_schema_for(&target, &env, column_name, current_value) + } + + /// [`Self::query_column_config_schema`] against an explicit target: the + /// column's group comes from the TARGET's static config, and its view type + /// from `view_schema` when given (a config not yet bound to a `View`), else + /// from the bound `View`'s schema. + fn query_column_config_schema_for( + &self, + target: &PluginTarget, + env: &ColumnSchemaEnv<'_>, + column_name: &str, + current_value: Option<&ConfigMap>, ) -> ApiResult { - let plugin = self.ensure_plugin_selected()?; - let plugin_config = self.metadata(); - let names = &plugin_config.config_column_names; + let ColumnSchemaEnv { + view_config, + session, + view_schema, + column_stats, + plugin_config, + } = *env; + + let names = &target.static_config.config_column_names; let group = view_config .columns .iter() @@ -423,115 +820,67 @@ impl Renderer { .and_then(|idx| names.get(idx)) .map(|s| s.as_str()); - if !session.metadata().has_view_schema() { - return Err(JsValue::from("view_schema not initialized").into()); - } - let Some(view_type) = session.metadata().get_column_view_type(column_name) else { + let view_type = match view_schema { + Some(view_schema) => view_schema.get(column_name).copied(), + None => { + if !session.metadata().has_view_schema() { + return Err(JsValue::from("view_schema not initialized").into()); + } + + session.metadata().get_column_view_type(column_name) + }, + }; + + let Some(view_type) = view_type else { return Ok(ColumnConfigSchema { fields: vec![] }); }; let current_js = JsValue::from_serde_ext(¤t_value).unwrap_or(JsValue::NULL); let view_config_js = JsValue::from_serde_ext(view_config).unwrap_or(JsValue::NULL); - let stats = session.get_column_stats(column_name).unwrap_or_default(); - let stats_json = serde_json::json!({ - "abs_max": stats.abs_max, - }); - let stats_js = JsValue::from_serde_ext(&stats_json).unwrap_or(JsValue::NULL); - - let raw = plugin._column_config_schema( + let plugin_config_js = JsValue::from_serde_ext(plugin_config).unwrap_or(JsValue::NULL); + let raw = target.element._column_config_schema( &view_type.to_string(), group, column_name, ¤t_js, &view_config_js, - &stats_js, + &plugin_config_js, )?; + let abs_max = column_stats + .then(|| { + session + .get_column_stats(column_name) + .and_then(|s| s.abs_max) + }) + .flatten(); + serde_wasm_bindgen::from_value::(raw) - .map(|schema| schema.canonicalize()) + .map(|schema| schema.canonicalize().resolve_stat_defaults(abs_max)) .map_err(|e| e.into()) } - /// Merge an update into the active plugin's plugin-level config map, - /// clearing entries equal to the schema default and dropping keys the - /// resulting schema no longer advertises. + /// Merge an update into the active plugin's plugin-level config map: + /// [`validate_plugin_config`] against the active target, then + /// [`Renderer::apply_plugin_config`]. pub fn update_plugin_config( &self, view_config: &ViewConfig, update: PluginConfigUpdate, ) -> ApiResult { - let Some(n) = self.active_plugin_name() else { + if self.active_plugin_name().is_none() { return Ok(false); - }; - - match update { - OptionalUpdate::SetDefault => { - let mut st = self.borrow_mut(); - let bucket = st.plugin_states.entry(n).or_default(); - let changed = !bucket.plugin.is_empty(); - bucket.plugin.clear(); - Ok(changed) - }, - OptionalUpdate::Missing => Ok(false), - OptionalUpdate::Update(mut map) => { - let mut merged = self.get_plugin_config(); - for (k, v) in &map { - merged.insert(k.clone(), v.clone()); - } - - let schema = self - .query_plugin_config_schema(view_config, Some(&merged)) - .ok(); - - let mut active = None; - if let Some(s) = &schema { - let keys = s.active_keys(); - map.retain(|k, _| keys.contains(k)); - active = Some(keys); - let errors = normalize_values(s, &mut map); - if let Some((key, error)) = errors.first() { - return Err(ApiError::from(JsValue::from_str(&format!( - "Invalid `plugin_config.{key}`: {error}" - )))); - } - } - - let mut st = self.borrow_mut(); - let bucket = st.plugin_states.entry(n).or_default(); - let mut changed = false; - if let Some(s) = &schema { - let leaves = s.leaf_fields(); - map.retain(|key, value| { - let is_default = leaves - .iter() - .any(|spec| matches_declared_default(spec, key, value)); - if is_default { - if bucket.plugin.remove(key).is_some() { - changed = true; - } - false - } else { - true - } - }); - } - - for (k, v) in map { - let prev = bucket.plugin.insert(k, v.clone()); - if prev.as_ref() != Some(&v) { - changed = true; - } - } + } - if let Some(active) = active { - let before = bucket.plugin.len(); - bucket.plugin.retain(|k, _| active.contains(k)); - changed |= bucket.plugin.len() != before; - } + let target = self.active_target()?; + let current = self.committed_plugin_config(); + let validated = validate_plugin_config(¤t, update, &|merged| { + Ok(self + .query_plugin_config_schema_for(&target, view_config, Some(merged)) + .ok()) + })?; - Ok(changed) - }, - } + Ok(self.apply_plugin_config(&target, validated)) } /// Apply a single schema-field update from the plugin-settings UI, @@ -545,7 +894,7 @@ impl Renderer { return false; }; - let mut next = self.get_plugin_config(); + let mut next = self.committed_plugin_config(); for k in &update.keys { match update.value.get(k) { Some(v) => { @@ -569,29 +918,28 @@ impl Renderer { strip_default_values(schema, &mut update.value); } - let mut st = self.borrow_mut(); - let bucket = st.plugin_states.entry(n).or_default(); - let mut changed = false; - - for k in &update.keys { - if let Some(v) = update.value.get(k) { - let prev = bucket.plugin.insert(k.to_string(), v.clone()); - if prev.as_ref() != Some(v) { + self.edit_bucket(&n, |bucket| { + let mut changed = false; + for k in &update.keys { + if let Some(v) = update.value.get(k) { + let prev = bucket.plugin.insert(k.to_string(), v.clone()); + if prev.as_ref() != Some(v) { + changed = true; + } + } else if bucket.plugin.remove(k).is_some() { changed = true; } - } else if bucket.plugin.remove(k).is_some() { - changed = true; } - } - if let Some(schema) = &schema { - let active = schema.active_keys(); - let before = bucket.plugin.len(); - bucket.plugin.retain(|k, _| active.contains(k)); - changed |= bucket.plugin.len() != before; - } + if let Some(schema) = &schema { + let active = schema.active_keys(); + let before = bucket.plugin.len(); + bucket.plugin.retain(|k, _| active.contains(k)); + changed |= bucket.plugin.len() != before; + } - changed + changed + }) } } @@ -833,6 +1181,7 @@ mod tests { min: None, max: None, step: None, + default_stat: None, }, ]; @@ -909,6 +1258,156 @@ mod tests { assert!(map.is_empty()); } + fn enum_schema() -> ColumnConfigSchema { + ColumnConfigSchema { + fields: vec![ + ControlSpec::Enum { + key: "fg_mode".to_owned(), + default: "color".to_owned(), + variants: vec![ + EnumVariant { + value: "disabled".to_owned(), + label: None, + }, + EnumVariant { + value: "color".to_owned(), + label: None, + }, + ], + }, + ControlSpec::Bool { + key: "flag".to_owned(), + default: false, + }, + ], + } + } + + fn obj(value: serde_json::Value) -> ConfigMap { + value.as_object().unwrap().clone() + } + + #[test] + fn validate_plugin_config_rejects_enum_values_outside_the_variants() { + let err = validate_plugin_config( + &ConfigMap::new(), + OptionalUpdate::Update(obj(json!({ "fg_mode": "series" }))), + &|_| Ok(Some(enum_schema())), + ) + .err() + .expect("rejected"); + + assert!( + err.0 + .contains("Invalid `plugin_config.fg_mode`: \"series\" is not one of"), + "{}", + err.0 + ); + } + + #[test] + fn validate_plugin_config_splits_defaults_and_prunes_to_active_keys() { + let validated = validate_plugin_config( + &obj(json!({ "fg_mode": "disabled" })), + OptionalUpdate::Update(obj(json!({ "fg_mode": "color", "flag": true, "ghost": 1 }))), + &|merged| { + assert_eq!(merged["fg_mode"], "color"); + assert_eq!(merged["flag"], true); + Ok(Some(enum_schema())) + }, + ) + .unwrap(); + + let ValidatedPluginConfig::Set { + map, + remove, + active, + } = validated + else { + panic!("expected Set"); + }; + + assert_eq!(map, obj(json!({ "flag": true }))); + assert_eq!(remove, vec!["fg_mode".to_owned()]); + assert_eq!( + active.unwrap(), + HashSet::from(["fg_mode".to_owned(), "flag".to_owned()]) + ); + } + + #[test] + fn validate_plugin_config_passes_through_without_a_schema() { + let update = obj(json!({ "fg_mode": "series", "ghost": 1 })); + let validated = validate_plugin_config( + &ConfigMap::new(), + OptionalUpdate::Update(update.clone()), + &|_| Ok(None), + ) + .unwrap(); + + assert_eq!(validated, ValidatedPluginConfig::Set { + map: update, + remove: vec![], + active: None, + }); + assert_eq!( + validate_plugin_config(&ConfigMap::new(), OptionalUpdate::SetDefault, &|_| { + Ok(Some(enum_schema())) + }) + .unwrap(), + ValidatedPluginConfig::Clear + ); + } + + #[test] + fn validate_columns_config_rejects_and_names_the_column() { + let err = validate_columns_config( + OptionalUpdate::Update(HashMap::from([( + "a".to_owned(), + obj(json!({ "fg_mode": "series" })), + )])), + &|_, _| Ok(Some(enum_schema())), + &|_| Some(ColumnType::Integer), + &|_, _| {}, + ) + .err() + .expect("rejected"); + + let msg = err.0; + assert!( + msg.contains("Invalid `columns_config[\"a\"].fg_mode`"), + "{msg}" + ); + assert!(msg.contains("for a integer column"), "{msg}"); + } + + #[test] + fn validate_columns_config_strips_defaults_and_keeps_unschematized_columns() { + let validated = validate_columns_config( + OptionalUpdate::Update(HashMap::from([ + ( + "a".to_owned(), + obj(json!({ "fg_mode": "color", "flag": true, "ghost": 1 })), + ), + ("b".to_owned(), obj(json!({ "anything": "goes" }))), + ])), + &|col, _| Ok((col == "a").then(enum_schema)), + &|_| None, + &|_, _| {}, + ) + .unwrap(); + + let ValidatedColumnsConfig::Set(mut entries) = validated else { + panic!("expected Set"); + }; + + entries.sort_by(|x, y| x.0.cmp(&y.0)); + assert_eq!(entries, vec![ + ("a".to_owned(), obj(json!({ "flag": true }))), + ("b".to_owned(), obj(json!({ "anything": "goes" }))), + ]); + } + #[test] fn normalize_values_canonicalizes_and_reports() { let schema = ColumnConfigSchema { diff --git a/rust/perspective-viewer/src/rust/session.rs b/rust/perspective-viewer/src/rust/session.rs index f1e5f676e1..33fb2d83c5 100644 --- a/rust/perspective-viewer/src/rust/session.rs +++ b/rust/perspective-viewer/src/rust/session.rs @@ -13,19 +13,23 @@ pub(crate) mod column_defaults_update; pub(crate) mod drag_drop_update; mod metadata; +mod op_queue; +mod panel_state; mod props; pub(crate) mod replace_expression_update; mod view_subscription; -use std::cell::{Cell, Ref, RefCell}; +use std::cell::{Cell, RefCell}; use std::collections::{HashMap, HashSet}; use std::future::Future; use std::ops::Deref; -use std::rc::{Rc, Weak}; +use std::rc::Rc; use perspective_client::config::*; use perspective_client::proto::ViewDimensionsResp; -use perspective_client::{Client, ClientError, ReconnectCallback, View}; +use perspective_client::{ + Client, ClientError, DescribeError, Description, ExprValidationResult, ReconnectCallback, View, +}; use perspective_js::apierror; use perspective_js::utils::*; use wasm_bindgen::prelude::*; @@ -34,6 +38,10 @@ use yew::prelude::*; use self::metadata::*; pub use self::metadata::{MetadataRef, SessionMetadata, SessionMetadataRc}; +use self::op_queue::OpQueue; +pub use self::op_queue::{EditDelta, OpCtx, OpKind, StepFuture, StepOutcome, Ticket, view_fields}; +use self::panel_state::effective; +pub use self::panel_state::{Binding, OverlayClause, PanelState, PluginRef}; pub use self::props::{SessionProps, TableLoadState}; pub use self::view_subscription::ViewStats; use self::view_subscription::*; @@ -96,10 +104,9 @@ pub struct SessionHandle { /// copying the config. pub last_dispatched_config: RefCell>>, - /// Open between a `load()` call and its payload's classification as - /// `Table`/`Client` ([`LoadWindow`]), held weakly so its [`LoadGuard`] is - /// the sole owner. - pending_load: RefCell>, + /// Every writer of this panel's committed state, run one at a time in + /// submit order (see [`op_queue`]). + queue: Rc, /// Coalesces `view_config_changed`: multiple synchronous commits in one /// task emit ONE event on the next microtask — the cadence the deleted @@ -187,29 +194,264 @@ pub(crate) async fn probe_table( /// Mutable state for `Session`. #[derive(Default)] pub struct SessionData { - client: Option, - table: Option, - pending_table: Option, - metadata: SessionMetadata, - config: ViewConfig, - global_filter: Vec, + /// This panel's committed state: ONE immutable value, replaced whole by + /// [`SessionData::swap`]. + state: Lifecycle, + + /// What [`Session::metadata`] answers while nothing is bound, and what + /// [`Session::metadata_mut`] scribbles on (every write is a no-op then). + unbound_metadata: Rc, view_sub: Option, stats: Option, - is_loading: bool, is_paused: bool, - /// Terminal: set by [`Session::mark_disposed`] when the owning panel is - /// ejected, never cleared. Distinct from an `Ejected` reset, which a - /// suspended or element-ejected session may legitimately rebind after. - disposed: Option, - - /// Memo for [`Session::validate_snapshot`]: the expression set validated - /// by the last successful server round trip; an equal snapshot skips the - /// round trip. Written only under the draw lock; cleared on table - /// (re)bind and expression reset. - last_validated_expressions: Option, - error: Option, - title: Option, + /// How the last config-driven run of the committed state went. + rendered: Rendered, +} + +/// A panel's state and whether it can still be written. +enum Lifecycle { + Live(Rc), + + /// Terminal: the owning panel was ejected ([`Session::dispose`]). + Disposed { + last: Rc, + disposal: Disposal, + }, +} + +impl Default for Lifecycle { + fn default() -> Self { + Self::Live(Rc::default()) + } +} + +impl Deref for Lifecycle { + type Target = Rc; + + fn deref(&self) -> &Self::Target { + match self { + Lifecycle::Live(state) | Lifecycle::Disposed { last: state, .. } => state, + } + } +} + +impl Lifecycle { + fn live_mut(&mut self) -> Option<&mut Rc> { + match self { + Lifecycle::Live(state) => Some(state), + Lifecycle::Disposed { .. } => None, + } + } + + fn disposal(&self) -> Option { + match self { + Lifecycle::Live(_) => None, + Lifecycle::Disposed { disposal, .. } => Some(*disposal), + } + } +} + +/// An OWNED read guard over the committed [`ViewConfig`]: a snapshot that +/// borrows nothing. +pub struct ViewConfigRef(Rc); + +impl Deref for ViewConfigRef { + type Target = ViewConfig; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +/// The [`crate::renderer::Renderer`]'s handle on its panel's committed +/// [`PanelState`] — read the current value, or swap in the next one. +#[derive(Clone)] +pub struct PanelCell(Session); + +impl PanelCell { + /// The COMMITTED state — what a running step reads. + pub fn state(&self) -> Rc { + self.0.borrow().state.clone() + } + + /// The theme as the UI sees it: the committed theme, or the latest pending + /// pick. + pub fn projected_theme(&self) -> Option { + let pending = self + .0 + .0 + .queue + .pending_edits() + .into_iter() + .filter_map(|delta| match delta { + EditDelta::Theme(theme) => Some(theme), + _ => None, + }) + .next_back(); + + pending.unwrap_or_else(|| self.state().chrome.theme.clone()) + } + + /// The SELECTED plugin's bucket as the UI and `save()` see it: the + /// committed bucket with every pending style / settings edit applied, in + /// submit order. + pub fn projected_bucket(&self, name: &str) -> crate::renderer::PluginScopedConfig { + let mut bucket = self.state().bucket(name); + for delta in self.0.0.queue.pending_edits() { + match delta { + EditDelta::PluginField(update) => { + for key in &update.keys { + match update.value.get(key) { + Some(value) => { + bucket.plugin.insert(key.clone(), value.clone()); + }, + None => { + bucket.plugin.remove(key); + }, + } + } + }, + EditDelta::ColumnField { column, update } => { + let entry = bucket.columns.entry(column.clone()).or_default(); + for key in &update.keys { + entry.remove(key); + } + + for (key, value) in update.value { + if update.keys.contains(&key) { + entry.insert(key, value); + } + } + + if entry.is_empty() { + bucket.columns.remove(&column); + } + }, + EditDelta::PluginConfig(map) => bucket.plugin.extend(map), + EditDelta::View(_) | EditDelta::Theme(_) | EditDelta::Title(_) => {}, + } + } + + bucket + } + + /// Submit a theme pick — a UI edit, committed by the drain in its turn. + pub fn submit_theme(&self, theme: Option) { + let cell = self.clone(); + let _ticket = self.0.submit( + OpKind::Edit { + delta: EditDelta::Theme(theme.clone()), + fields: None, + }, + move |_ctx| { + Box::pin(async move { + cell.swap(cell.state().with_theme(theme)); + Ok(StepOutcome::Done) + }) + }, + ); + } + + /// Submit an op on this panel's queue (see [`Session::submit`]). + pub fn submit(&self, kind: OpKind, step: impl FnOnce(OpCtx) -> StepFuture + 'static) -> Ticket { + self.0.submit(kind, step) + } + + pub fn swap(&self, next: PanelState) { + self.0.borrow_mut().swap(next); + } +} + +/// The outcome of the last config-driven run. +#[derive(Clone, Default)] +pub enum Rendered { + #[default] + Never, + Ok, + + /// The run of THIS state failed. + Failed(Rc, TableErrorState), +} + +impl SessionData { + fn error(&self) -> Option<&TableErrorState> { + self.state.lost().or(match &self.rendered { + Rendered::Failed(_, error) => Some(error), + _ => None, + }) + } + + fn clear_errors(&mut self) { + if self.state.lost().is_some() { + self.swap(self.state.recovered()); + } + + if matches!(self.rendered, Rendered::Failed(..)) { + self.rendered = Rendered::Never; + } + } + + fn swap(&mut self, next: PanelState) { + if let Some(state) = self.state.live_mut() { + *state = Rc::new(next); + } + } + + fn table(&self) -> Option<&perspective_client::Table> { + self.state.bound().map(|bound| &bound.table) + } + + fn pending_table(&self) -> Option<&str> { + self.state.awaiting() + } + + fn metadata(&self) -> &Rc { + match self.state.bound() { + Some(bound) => &bound.metadata, + None => &self.unbound_metadata, + } + } + + /// Copy-on-write: a snapshot held elsewhere never observes the mutation. + fn metadata_mut(&mut self) -> &mut SessionMetadata { + let Some(state) = self.state.live_mut() else { + return Rc::make_mut(&mut self.unbound_metadata); + }; + + let binding = match &mut Rc::make_mut(state).binding { + Binding::Lost { prior, .. } => Rc::make_mut(prior), + binding => binding, + }; + + match binding { + Binding::Bound(bound) => Rc::make_mut(&mut Rc::make_mut(bound).metadata), + _ => Rc::make_mut(&mut self.unbound_metadata), + } + } + + fn config(&self) -> &ViewConfig { + self.state.config() + } + + fn set_config(&mut self, config: ViewConfig) { + self.swap(self.state.with_config(Rc::new(config))); + } + + /// Memo for [`Session::validate_snapshot`]: the [`Description`] the server + /// gave for the last validated effective config; an equal snapshot skips + /// the round trip. + fn description(&self) -> Option<&(Rc, Rc)> { + self.state.bound()?.description.as_ref() + } + + fn set_description(&mut self, description: Option<(Rc, Rc)>) { + self.swap(self.state.with_description(description)); + } + + fn unbind(&mut self) { + self.swap(self.state.unbound()); + } } #[derive(Clone)] @@ -260,59 +502,19 @@ pub enum Disposal { Resolve, } -#[derive(Debug, Default)] -pub enum TableIntermediateState { - #[default] - Ejected, - Reloaded, -} - -/// The raw config deltas committed while a `load()` classifies its payload, in -/// commit order, awaiting replay over a reset base if it classifies as a -/// `Table` — identified by [`Rc`] identity rather than a generation counter. -#[derive(Default)] -pub struct LoadWindow { - journal: RefCell>, -} - -/// A `load()` call site's handle on THE window it opened, closing that window -/// on `Drop` so no exit can strand `is_loading`. -pub struct LoadGuard { - session: Session, - window: Rc, -} - -impl LoadGuard { - /// Close this window and take its journal for replay, or `None` when a - /// later `load()` superseded it — "abandon this classification, hold the - /// frame". - pub fn claim(&self) -> Option> { - self.close_current().then(|| self.window.journal.take()) - } - - /// Close this window, discarding its journal; idempotent, and a no-op once - /// superseded. - pub fn close(&self) { - self.close_current(); - } - - fn close_current(&self) -> bool { - let mut slot = self.session.pending_load.borrow_mut(); - if !slot.upgrade().is_some_and(|x| Rc::ptr_eq(&x, &self.window)) { - return false; +impl From for ApiResult<()> { + fn from(disposal: Disposal) -> Self { + match disposal { + Disposal::Reject => Err(ApiError::new("Panel disposed")), + Disposal::Resolve => Ok(()), } - - *slot = Weak::new(); - drop(slot); - self.session.borrow_mut().is_loading = false; - true } } -impl Drop for LoadGuard { - fn drop(&mut self) { - self.close(); - } +#[derive(Debug, Default)] +pub enum TableIntermediateState { + #[default] + Ejected, } /// Options for [`Session::reset`] @@ -361,21 +563,62 @@ impl Session { Self(Rc::default()) } - pub(crate) fn metadata(&self) -> MetadataRef<'_> { - std::cell::Ref::map(self.borrow(), |x| &x.metadata) + pub(crate) fn metadata(&self) -> MetadataRef { + MetadataRef(self.borrow().metadata().clone()) + } + + /// A handle on this panel's committed [`PanelState`] for the panel's + /// [`crate::renderer::Renderer`], which owns none of it. + pub fn cell(&self) -> PanelCell { + PanelCell(self.clone()) } pub(crate) fn metadata_mut(&self) -> MetadataMutRef<'_> { - std::cell::RefMut::map(self.borrow_mut(), |x| &mut x.metadata) + std::cell::RefMut::map(self.borrow_mut(), |x| x.metadata_mut()) } + /// The title as the UI and `save()` see it: the committed title, or the + /// latest pending rename. pub(crate) fn get_title(&self) -> Option { - self.borrow().title.clone() + let pending = self + .0 + .queue + .pending_edits() + .into_iter() + .filter_map(|delta| match delta { + EditDelta::Title(title) => Some(title), + _ => None, + }) + .next_back(); + + pending.unwrap_or_else(|| self.borrow().state.chrome.title.clone()) } + /// Rename this panel — a UI edit, committed by the drain in its turn. pub fn set_title(&self, title: Option) { + let title = title.filter(|x| !x.is_empty()); + let session = self.clone(); + let _ticket = self.submit( + OpKind::Edit { + delta: EditDelta::Title(title.clone()), + fields: None, + }, + move |_ctx| { + Box::pin(async move { + session.commit_title(title); + Ok(StepOutcome::Done) + }) + }, + ); + + self.title_changed.emit(self.get_title()); + } + + /// Write the title NOW. + pub(crate) fn commit_title(&self, title: Option) { let new_title = title.filter(|x| !x.is_empty()); - self.borrow_mut().title.clone_from(&new_title); + let next = self.borrow().state.with_title(new_title.clone()); + self.borrow_mut().swap(next); self.title_changed.emit(new_title); } @@ -385,37 +628,26 @@ impl Session { /// `Session`. pub fn reset(&self, options: ResetOptions) -> impl Future> + use<> { let view = self.0.borrow_mut().view_sub.take(); - let had_table = self.borrow().table.is_some(); + let had_table = self.borrow().table().is_some(); let err = self.get_error(); - self.borrow_mut().error = None; + self.borrow_mut().clear_errors(); if options.stats { self.update_stats(ViewStats::default()); } if options.config { - self.borrow_mut().config.reset(options.expressions); + let mut config = self.borrow().config().clone(); + config.reset(options.expressions); + self.borrow_mut().set_config(config); } if options.expressions { - self.borrow_mut().last_validated_expressions = None; + self.borrow_mut().set_description(None); } match options.table { - Some(TableIntermediateState::Ejected) => { - self.borrow_mut().is_loading = false; - self.borrow_mut().table = None; - self.borrow_mut().pending_table = None; - self.borrow_mut().metadata = SessionMetadata::default(); - }, - Some(TableIntermediateState::Reloaded) => { - self.borrow_mut().is_loading = true; - self.borrow_mut().table = None; - self.borrow_mut().pending_table = None; - self.borrow_mut().metadata = SessionMetadata::default(); - }, - _ => { - self.borrow_mut().is_loading = false; - }, + Some(TableIntermediateState::Ejected) => self.borrow_mut().unbind(), + None => {}, }; // A config reset that KEEPS its `Table` is itself a commit, and every @@ -440,52 +672,82 @@ impl Session { } } - /// Open a load window (see [`LoadWindow`]) at the `load()` call site, - /// returning the [`LoadGuard`] that owns it. Sets `is_loading` so a FRESH - /// panel shows the spinner while a slow payload resolves (a RELOAD keeps - /// its bound table, hence [`TableLoadState::Loaded`], throughout — no - /// spinner flicker). - pub fn begin_pending_load(&self) -> Rc { - let window = Rc::new(LoadWindow::default()); - *self.0.pending_load.borrow_mut() = Rc::downgrade(&window); - self.borrow_mut().is_loading = true; - Rc::new(LoadGuard { - session: self.clone(), - window, - }) + /// Submit a writer of this panel's state. + pub fn submit(&self, kind: OpKind, step: impl FnOnce(OpCtx) -> StepFuture + 'static) -> Ticket { + if let Some(disposal) = self.disposal() { + return Ticket::settled(disposal.into()); + } + + let is_edit = matches!(kind, OpKind::Edit { + delta: EditDelta::View(_), + .. + }); + + let guard = self.begin_config_run(); + let (ticket, start) = self.0.queue.push(kind, guard, step); + if is_edit { + self.notify_view_config_changed(); + } + + if start { + let queue = self.0.queue.clone(); + let session = self.clone(); + ApiFuture::spawn_named("op-queue-drain", async move { + queue + .drain(move |_exit| session.notify_view_config_changed()) + .await; + + Ok(()) + }); + } + + ticket } - /// Whether a `load()` payload is still awaiting classification. A `true` - /// value makes every config-driven bind DEFER - /// ([`crate::tasks::bind_snapshot`]) — the incoming config must never draw - /// against the outgoing table. + /// Resolves once every op submitted to this panel has settled — its write + /// made (or rejected) AND its render landed. + pub async fn settle_ops(&self) { + self.0.queue.idle().await + } + + /// Whether a `load()` is queued or running. pub fn has_pending_load(&self) -> bool { - self.0.pending_load.borrow().strong_count() > 0 + self.0.queue.has_load() } - /// Mark this session disposed (its panel was ejected): every later - /// table bind and locked run refuses, so nothing can rebind or draw a - /// panel that no longer exists. - pub(crate) fn mark_disposed(&self, disposal: Disposal) { - self.borrow_mut().disposed = Some(disposal); + /// The ONE `Live → Disposed` transition (this session's panel was ejected): + /// the state is frozen as its last value, everything queued settles per + /// `disposal`, and every later `submit` settles at once. + pub(crate) fn dispose(&self, disposal: Disposal) { + { + let mut data = self.borrow_mut(); + if let Lifecycle::Live(last) = &data.state { + data.state = Lifecycle::Disposed { + last: last.clone(), + disposal, + }; + } + } + + self.0.queue.reject_all(disposal.into()); } pub(crate) fn is_disposed(&self) -> bool { - self.borrow().disposed.is_some() + self.disposal().is_some() } /// This session's disposal, if its panel was ejected. pub(crate) fn disposal(&self) -> Option { - self.borrow().disposed + self.borrow().state.disposal() } pub(crate) fn has_table(&self) -> Option { let data = self.borrow(); - if data.table.is_some() { + if data.table().is_some() { Some(TableLoadState::Loaded) - } else if data.is_loading { + } else if self.0.queue.has_load() { Some(TableLoadState::Loading) - } else if data.pending_table.is_some() { + } else if data.pending_table().is_some() { Some(TableLoadState::Pending) } else { None @@ -493,13 +755,15 @@ impl Session { } pub fn get_table(&self) -> Option { - self.borrow().table.clone() + self.borrow().table().cloned() } + /// Look this panel's tables up on `client` — unbinding it from a table of + /// any other client's. pub fn set_client(&self, client: Client) -> bool { - if Some(&client) != self.borrow().client.as_ref() { - self.borrow_mut().client = Some(client); - self.borrow_mut().table = None; + if Some(&client) != self.get_client().as_ref() { + let next = self.borrow().state.unbound_on(Some(client)); + self.borrow_mut().swap(next); true } else { false @@ -507,13 +771,13 @@ impl Session { } pub fn get_client(&self) -> Option { - self.borrow().client.clone() + self.borrow().state.client() } /// The configured table name awaiting a host, if any (see /// [`SessionData::pending_table`]). pub fn pending_table(&self) -> Option { - self.borrow().pending_table.clone() + self.borrow().pending_table().map(str::to_owned) } /// Suspend a BOUND table to PENDING: delete the `View`, drop the `Table` @@ -523,64 +787,35 @@ impl Session { /// as it was when the name is hosted again. `None` when no table is /// bound. pub fn suspend_table(&self) -> Option> + use<>> { - let name = self.borrow().table.as_ref()?.get_name().to_owned(); + let name = self.borrow().table()?.get_name().to_owned(); + let client = self.get_client()?; let fut = self.reset(ResetOptions { table: Some(TableIntermediateState::Ejected), stats: true, ..ResetOptions::default() }); - self.borrow_mut().pending_table = Some(name); + let next = self.borrow().state.awaiting_table(client, name); + self.borrow_mut().swap(next); Some(fut) } - /// Reset this `Session`'s state with a new `Table`. Implicitly clears the - /// `ViewSubscription`, which will need to be re-initialized later via - /// `create_view()`. - /// - /// # Arguments - /// - /// - `table_name` The name of the `Table` to load. - /// - /// # Returns - /// - /// `table_name` is unique per `Client`, so if this value has not changed, - /// `Session::set_table` does nothing and returns `Ok(false)`. - /// - /// A name no loaded client hosts pends ([`MissingTable::Pend`]) and - /// returns `Ok(false)` until the table lifecycle re-runs the bind. - pub async fn set_table(&self, table_name: String) -> ApiResult { - if Some(table_name.as_str()) == self.0.borrow().table.as_ref().map(|x| x.get_name()) { - self.0.borrow_mut().pending_table = None; - return Ok(false); - } - - let client = self.0.borrow().client.clone().into_apierror()?; - match probe_table(&client, &table_name, MissingTable::Pend).await? { - Some(table) => match self.bind_table(table).await { - Ok(()) => Ok(true), - Err(err) => self.set_error(false, err).await.map(|_| false), - }, - None => { - self.pend_table(table_name).await?; - Ok(false) - }, - } - } - - /// Bind an opened `Table`, replacing any previous binding and announcing - /// `table_loaded`. - pub(crate) async fn bind_table(&self, table: perspective_client::Table) -> ApiResult<()> { - if self.is_disposed() { - return Err(ApiError::new("Panel disposed")); - } - - let metadata = SessionMetadata::from_table(&table).await?; - let client = table.get_client(); + /// Record a connection error on `client` as this panel's lost binding — for + /// as long as `client` is the one the panel is bound through. + async fn watch_client(&self, client: &Client) -> ApiResult<()> { let on_error = self.on_table_errored.borrow().clone(); let session = self.clone(); + let watched = client.clone(); let poll_loop = LocalPollLoop::new(move |(message, reconnect): (ApiError, _)| { - session.borrow_mut().error = Some(TableErrorState(message, reconnect)); + if session.get_client().as_ref() != Some(&watched) { + return Ok(JsValue::UNDEFINED); + } + + let next = session + .borrow() + .state + .with_lost(TableErrorState(message, reconnect)); + session.borrow_mut().swap(next); if let Some(cb) = &on_error { cb.emit(()); } @@ -601,33 +836,9 @@ impl Session { })) .await?; - let sub = self.borrow_mut().view_sub.take(); - self.borrow_mut().metadata = metadata; - self.borrow_mut().table = Some(table); - self.borrow_mut().pending_table = None; - self.borrow_mut().is_loading = false; - self.borrow_mut().last_validated_expressions = None; - sub.delete().await?; - self.table_loaded.emit(()); Ok(()) } - /// Record `name` as pending, dropping any bound `Table` so nothing binds - /// against the outgoing table while the name waits for a host. - pub(crate) async fn pend_table(&self, name: String) -> ApiResult<()> { - if self.is_disposed() { - return Err(ApiError::new("Panel disposed")); - } - - let sub = self.borrow_mut().view_sub.take(); - self.borrow_mut().table = None; - self.borrow_mut().metadata = SessionMetadata::default(); - self.borrow_mut().pending_table = Some(name); - self.borrow_mut().is_loading = false; - self.borrow_mut().last_validated_expressions = None; - sub.delete().await - } - pub async fn set_error(&self, reset_table: bool, err: ApiError) -> ApiResult<()> { let session = self.clone(); let poll_loop = LocalPollLoop::new(move |()| { @@ -639,7 +850,7 @@ impl Session { Ok(JsValue::UNDEFINED) }); - self.borrow_mut().error = Some(TableErrorState( + let error = TableErrorState( err.clone(), Some(ReconnectCallback::new(move || { clone!(poll_loop); @@ -648,7 +859,10 @@ impl Session { Ok(()) }) })), - )); + ); + + let next = self.borrow().state.with_lost(error); + self.borrow_mut().swap(next); if let Some(cb) = self.on_table_errored.borrow().as_ref() { cb.emit(()); @@ -656,7 +870,7 @@ impl Session { let sub = self.borrow_mut().view_sub.take(); if reset_table { - self.borrow_mut().table = None; + self.borrow_mut().unbind(); } sub.delete().await?; @@ -686,24 +900,61 @@ impl Session { } pub fn js_get_table(&self) -> Option { - Some(perspective_js::Table::from(self.borrow().table.clone()?).into()) + Some(perspective_js::Table::from(self.borrow().table().cloned()?).into()) } + /// Whether the binding is lost OR the last run failed. pub(crate) fn is_errored(&self) -> bool { - self.borrow().error.is_some() + self.borrow().error().is_some() } pub(crate) fn get_error(&self) -> Option { - self.borrow().error.as_ref().map(|x| x.0.clone()) + self.borrow().error().map(|x| x.0.clone()) + } + + /// The error a config-driven run must not proceed past: a lost binding, or + /// a failed run of EXACTLY the state now committed (re-running it would + /// only fail again). + pub(crate) fn blocking_error(&self) -> Option { + let data = self.borrow(); + if let Some(error) = data.state.lost() { + return Some(error.0.clone()); + } + + match &data.rendered { + Rendered::Failed(state, error) if Rc::ptr_eq(state, &data.state) => { + Some(error.0.clone()) + }, + _ => None, + } } + /// Record that a config-driven run of the committed state is starting (any + /// earlier failure is stale) or has landed. + pub(crate) fn set_rendered(&self, ok: bool) { + self.borrow_mut().rendered = if ok { Rendered::Ok } else { Rendered::Never }; + } + + /// Recover from an error state (the overlay's button) — a writer, so an op + /// on the queue like any other. pub async fn reconnect(&self) -> ApiResult<()> { - let err = self.borrow().error.clone(); + let session = self.clone(); + self.submit(OpKind::Restore { fields: None }, move |_ctx| { + Box::pin(async move { + session.reconnect_step().await?; + Ok(StepOutcome::Done) + }) + }) + .settle() + .await + } + + async fn reconnect_step(&self) -> ApiResult<()> { + let err = self.borrow().error().cloned(); if let Some(TableErrorState(_, Some(reconnect))) = err { reconnect().await?; - self.borrow_mut().is_loading = false; - self.borrow_mut().error = None; - self.borrow_mut().last_validated_expressions = None; + self.borrow_mut().clear_errors(); + self.borrow_mut().set_description(None); self.borrow_mut().view_sub = None; self.table_loaded.emit(()); } @@ -739,36 +990,93 @@ impl Session { .is_some_and(|s| s.num_table_cells.is_some()) } - pub fn get_view_config(&'_ self) -> Ref<'_, ViewConfig> { - Ref::map(self.borrow(), |x| &x.config) + /// The view config as the UI and `save()` see it: the committed config with + /// every pending UI edit applied, in submit order. + pub fn get_view_config(&self) -> ViewConfigRef { + ViewConfigRef(self.projected()) } - /// The effective [`ViewConfig`] the `View` is built from: the stored config - /// with the transient element-level [`SessionData::global_filter`] clauses - /// appended. Used at view-creation only; the stored `config` (hence - /// `savePanel`/the settings UI) is unaffected. - fn effective_view_config(&self) -> ViewConfig { - let data = self.borrow(); - if data.global_filter.is_empty() { - return data.config.clone(); + /// The COMMITTED view config — what a running op must read, since the edits + /// still queued behind it have not happened yet. + pub fn committed_view_config(&self) -> ViewConfigRef { + ViewConfigRef(self.borrow().state.config().clone()) + } + + fn projected(&self) -> Rc { + let committed = self.borrow().state.config().clone(); + let edits = self + .0 + .queue + .pending_edits() + .into_iter() + .filter_map(|delta| match delta { + EditDelta::View(delta) => Some(*delta), + _ => None, + }) + .collect::>(); + + if edits.is_empty() { + return committed; } - let mut config = data.config.clone(); - config.filter.extend(data.global_filter.iter().cloned()); - config + let mut config = (*committed).clone(); + for delta in edits { + let mut candidate = config.clone(); + if candidate.apply_update(delta) && self.validate_names(&candidate).is_ok() { + self.normalize_view_config(&mut candidate); + config = candidate; + } + } + + Rc::new(config) } - /// Replace the transient element-level global filters and re-render the - /// view (no-op if unchanged). These are applied on top of the panel's - /// own config at view-creation but never persisted into it. - pub fn set_global_filter(&self, filter: Vec) -> bool { - if self.borrow().global_filter != filter { - self.borrow_mut().global_filter = filter; - self.notify_view_config_changed(); - true - } else { - false + /// Whether `delta` is an edit the UI may submit: SYNCHRONOUS validation + /// against the projection, so an invalid edit is refused at the control + /// that made it. + pub fn check_edit(&self, delta: &ViewConfigUpdate) -> ApiResult<()> { + if let Some(x) = self.borrow().state.lost() { + return Err(ApiError::new(x.0.clone())); + } + + let mut candidate = (*self.projected()).clone(); + if candidate.apply_update(delta.clone()) { + self.validate_names(&candidate)?; + } + + Ok(()) + } + + /// The effective [`ViewConfig`] the `View` is built from — the one the + /// committed description describes. + fn effective_view_config(&self) -> Rc { + let data = self.borrow(); + if let Some((effective, _)) = data.description() { + return effective.clone(); } + + let metadata = data.metadata().clone(); + let (config, _) = effective(data.config(), &data.state.overlay, &|name| { + metadata.get_column_table_type(name) + }); + + Rc::new(config) + } + + /// The overlay clauses this panel's table cannot honor, by index — a column + /// it lacks, or has with another type. + pub fn skipped_overlay(&self) -> Vec { + let data = self.borrow(); + let metadata = data.metadata().clone(); + effective(data.config(), &data.state.overlay, &|name| { + metadata.get_column_table_type(name) + }) + .1 + } + + /// The element's global filter as last broadcast to this panel. + pub(crate) fn committed_overlay(&self) -> Rc> { + self.borrow().state.overlay.clone() } /// Snapshot of the [`ViewConfig`] the currently-bound `View` was @@ -777,9 +1085,8 @@ impl Session { /// /// Prefer this over [`Self::get_view_config`] when you need a /// value consistent with what the active plugin is rendering. - /// `get_view_config` returns the live config, which is mutated - /// synchronously by [`Self::commit_view_config`] ahead of the next - /// queued run and so may temporarily disagree with the bound `View`. + /// `get_view_config` returns the PROJECTED config — committed state plus + /// pending edits — which runs ahead of the bound `View`. pub fn get_rendered_view_config(&self) -> Option> { self.borrow().view_sub.as_ref().map(|s| s.get_view_config()) } @@ -790,7 +1097,7 @@ impl Session { config_static: &PluginStaticConfig, ) { use self::column_defaults_update::*; - let config = self.get_view_config(); + let config = self.committed_view_config(); config_update.set_update_column_defaults( &self.metadata(), &config, @@ -812,72 +1119,18 @@ impl Session { use self::column_defaults_update::*; config_update.set_update_rollup_defaults( &self.metadata(), - &self.get_view_config(), + &self.committed_view_config(), config_static, ) } - /// Apply a `ViewConfigUpdate` to the live config — the ONLY view-config - /// mutator (invariant I1: synchronous and total; no `await` separates any - /// read of the config from this write, so a lost update is - /// unrepresentable). - /// - /// Validation is SYNCHRONOUS and happens before anything is applied: an - /// update naming an unknown column is rejected with `Err` and the config - /// is untouched (I4 — invalid state is never entered, so no rollback - /// path exists). Server-side expression compilation is deliberately NOT - /// checked here; it is a property of a pipeline run - /// ([`Self::validate_snapshot`]) and fails that run, never the commit. - pub fn commit_view_config(&self, config_update: ViewConfigUpdate) -> ApiResult<()> { - if let Some(x) = self.borrow().error.as_ref() { - tracing::warn!("Errored state"); - - // Load bearing return - return Err(ApiError::new(x.0.clone())); - } - - // A `load()` is classifying its payload (see - // [`SessionHandle::pending_load`]): record the RAW delta so a `Table` - // classification can replay it over the reset base, validated against - // the INCOMING table's schema. The live apply below is a best-effort - // preview (kept for `save()` coherence, I1) that SKIPS name validation - // — the delta may legitimately name the incoming table's columns, - // absent from the outgoing one — mirroring the "no table bound yet" - // leniency already in `validate_names`. Authoritative validation is - // deferred to the replay. - let window = self.0.pending_load.borrow().upgrade(); - let mut candidate = self.borrow().config.clone(); - let journal_entry = window.is_some().then(|| config_update.clone()); - if !candidate.apply_update(config_update) { - return Ok(()); - } - - if let Some(window) = &window - && let Some(entry) = journal_entry - { - window.journal.borrow_mut().push(entry); - } - - if window.is_none() { - self.validate_names(&candidate)?; - } - - self.normalize_view_config(&mut candidate); - self.borrow_mut().config = candidate; - self.notify_view_config_changed(); - Ok(()) - } - - /// Table-bind commit: normalize the (possibly empty) config against the - /// newly-bound table's metadata — the default-view materialization that - /// previously happened inside the async validate write-back. SYNC; - /// called immediately after `set_table().await` inside the binding run, - /// so it is ordered like any other commit. - pub fn commit_table_defaults(&self) { - let mut candidate = self.borrow().config.clone(); + /// Re-normalize the config against the bound table's metadata — the + /// default-view materialization after a config reset that keeps its table. + fn commit_table_defaults(&self) { + let mut candidate = self.borrow().config().clone(); self.normalize_view_config(&mut candidate); - if candidate != self.borrow().config { - self.borrow_mut().config = candidate; + if candidate != *self.borrow().config() { + self.borrow_mut().set_config(candidate); self.notify_view_config_changed(); } } @@ -889,7 +1142,13 @@ impl Session { /// when no table is bound yet: the config rides along until `load()` /// binds one, and the engine surfaces any residual error on that run. fn validate_names(&self, config: &ViewConfig) -> ApiResult<()> { - let table_columns = self.all_columns(); + Self::validate_names_with(&self.metadata(), config) + } + + /// [`Self::validate_names`] against an explicit table's metadata — the + /// INCOMING table's, for a restore that binds one. + fn validate_names_with(metadata: &SessionMetadata, config: &ViewConfig) -> ApiResult<()> { + let table_columns = Self::columns_of(metadata); if table_columns.is_empty() { return Ok(()); } @@ -932,7 +1191,11 @@ impl Session { /// write-back): fill empty `columns` from the table, prune `aggregates` /// to referenced columns. fn normalize_view_config(&self, config: &mut ViewConfig) { - let table_columns = self.all_columns(); + Self::normalize_with(&self.metadata(), config) + } + + fn normalize_with(metadata: &SessionMetadata, config: &mut ViewConfig) { + let table_columns = Self::columns_of(metadata); if table_columns.is_empty() { return; } @@ -1038,53 +1301,335 @@ impl Session { /// that preceded run *N*'s completion (invariant I3). pub fn snapshot(&self, _guard: &RenderGuard) -> ConfigSnapshot { ConfigSnapshot { - config: Rc::new(self.borrow().config.clone()), - effective: Rc::new(self.effective_view_config()), + config: self.borrow().state.config().clone(), + effective: self.effective_view_config(), } } - /// Validate a snapshot's expressions against the server (the only - /// inherently-async validation), updating the metadata expression - /// schema. Reads nothing from — and writes nothing to — the live - /// config; a failure fails this RUN, never the committed config (I4). - /// - /// Memoized: when the snapshot's expressions equal the last successfully - /// validated set, the round trip is skipped entirely — safe because the - /// memo key is an immutable snapshot field and the recorded set is - /// written only under the draw lock. + /// Validate a snapshot against the server with `Table::describe`, failing + /// this run and never the committed config. pub async fn validate_snapshot( &self, _guard: &RenderGuard, snap: ConfigSnapshot, ) -> ApiResult { - let memo_hit = - self.borrow().last_validated_expressions.as_ref() == Some(&snap.effective.expressions); - if !memo_hit { - let supports_expressions = self - .metadata() - .get_features() - .map(|x| x.expressions) - .unwrap_or_default(); - - if supports_expressions { - let table = self - .borrow() - .table - .as_ref() - .ok_or_else(|| apierror!(NoTableError))? - .clone(); - - let valid_recs = table - .validate_expressions(snap.effective.expressions.clone()) - .await?; - - self.metadata_mut().update_expressions(&valid_recs)?; + let (description, fresh) = self.describe_effective(&snap.effective).await?; + if fresh { + tracing::warn!("Rendering a commit that was not described"); + Self::record_description(&mut self.metadata_mut(), &snap.effective, &description)?; + self.borrow_mut() + .set_description(Some((snap.effective.clone(), description.clone()))); + } + + Ok(ValidatedSnapshot { snap }) + } + + /// The server's [`Description`] of `effective`, and whether it took a round + /// trip (`false` when the memo already held it). + async fn describe_effective( + &self, + effective: &Rc, + ) -> ApiResult<(Rc, bool)> { + let memo = self + .borrow() + .description() + .cloned() + .filter(|(key, _)| **key == **effective) + .map(|(_, description)| description); + + if let Some(description) = memo { + return Ok((description, false)); + } + + let table = self + .borrow() + .table() + .cloned() + .ok_or_else(|| apierror!(NoTableError))?; + + Self::describe_with(&table, &self.metadata(), effective) + .await + .map(|description| (description, true)) + } + + /// `table`'s [`Description`] of `effective`, with validation failures as + /// the errors a `restore()` rejects with. + async fn describe_with( + table: &perspective_client::Table, + metadata: &SessionMetadata, + effective: &Rc, + ) -> ApiResult> { + let engine_config = Self::with_default_aggregates_of(metadata, effective); + match table.describe(engine_config.into()).await? { + Ok(description) => Ok(Rc::new(description)), + Err(DescribeError::Expressions { + expression_schema, + errors, + }) => Err(apierror!(InvalidViewerConfigExpressionsError(Rc::new( + ExprValidationResult { + expression_schema, + errors, + expression_alias: effective.expressions.0.clone(), + } + )))), + Err(DescribeError::Config(msg)) => Err(ApiError::new(msg)), + } + } + + /// The PREPARE half of a transactional restore's view of the panel: the + /// binding `plan` resolved, and the config it leaves — the committed one (a + /// default one, for a plan that resets) with `update` applied — + /// name-checked, normalized and DESCRIBED by the table it will be bound to. + pub(crate) async fn prepare_view( + &self, + plan: BindPlan, + mut update: ViewConfigUpdate, + defaults: ViewDefaults<'_>, + overlay: Option>>, + ) -> ApiResult { + if self.is_disposed() { + return Err(ApiError::new("Panel disposed")); + } + + if matches!(plan, BindPlan::Keep) + && let Some(error) = self.borrow().state.lost() + { + return Err(error.0.clone()); + } + + let committed = self.borrow().state.config().clone(); + let base = match &plan { + BindPlan::Bind { reset: true, .. } | BindPlan::Pend { reset: true, .. } => { + Rc::new(ViewConfig::default()) + }, + _ => committed.clone(), + }; + + let (binding, checked) = match plan { + BindPlan::Keep => { + let bound = self.borrow().state.bound().cloned(); + ( + PreparedBinding::Keep, + bound.map(|bound| (bound.table.clone(), bound.metadata.clone())), + ) + }, + BindPlan::Bind { client, table, .. } => { + let metadata = Rc::new(SessionMetadata::from_table(&table).await?); + let binding = PreparedBinding::Bind { + client, + table: table.clone(), + metadata: metadata.clone(), + }; + + (binding, Some((*table, metadata))) + }, + BindPlan::Pend { client, name, .. } => (PreparedBinding::Pend { client, name }, None), + }; + + { + use self::column_defaults_update::*; + let metadata = match &checked { + Some((_, metadata)) => metadata.clone(), + None => self.borrow().metadata().clone(), + }; + + match defaults { + ViewDefaults::Swap(plugin) => { + update.set_update_column_defaults(&metadata, &base, &base.columns, plugin) + }, + ViewDefaults::Rollup(plugin) => { + update.set_update_rollup_defaults(&metadata, &base, plugin) + }, + ViewDefaults::AsGiven => {}, + } + } + + let mut candidate = (*base).clone(); + candidate.apply_update(update); + if let Some((_, metadata)) = &checked { + Self::validate_names_with(metadata, &candidate)?; + Self::normalize_with(metadata, &mut candidate); + } + + let config = if candidate == *committed { + committed.clone() + } else { + Rc::new(candidate) + }; + + let changed = !Rc::ptr_eq(&config, &committed); + let clauses = overlay + .clone() + .unwrap_or_else(|| self.borrow().state.overlay.clone()); + + let (effective, description) = match &checked { + None => (config.clone(), None), + Some((table, metadata)) => { + let on_expression = clauses + .iter() + .any(|x| config.expressions.0.contains_key(x.filter.column())); + + let bare = if on_expression { + Some(Self::describe_with(table, metadata, &config).await?) + } else { + None + }; + + let (effective, _) = effective(&config, &clauses, &|name| { + metadata.get_table_schema_type(name).or_else(|| { + bare.as_ref() + .and_then(|x| x.expression_schema.get(name).copied()) + }) + }); + + let effective = if effective == *config { + config.clone() + } else { + Rc::new(effective) + }; + + let description = match (&binding, bare) { + (_, Some(bare)) if Rc::ptr_eq(&effective, &config) => bare, + (PreparedBinding::Keep, _) => self.describe_effective(&effective).await?.0, + _ => Self::describe_with(table, metadata, &effective).await?, + }; + + (effective, Some(description)) + }, + }; + + Ok(PreparedView { + binding, + overlay, + config, + effective, + description, + changed, + announce: true, + }) + } + + /// The COMMIT half of [`Self::prepare_view`], as ONE swap: `state` with the + /// prepared binding, config, description and expression metadata, then + /// whatever else of the panel `rest` replaces. + pub(crate) fn commit_view( + &self, + view: PreparedView, + rest: impl FnOnce(PanelState) -> PanelState, + ) -> BindingEffects { + let state = self.borrow().state.clone(); + let title_before = state.chrome.title.clone(); + let had_table = state.bound().is_some(); + let (rebound, watch) = match view.binding { + PreparedBinding::Keep => (None, None), + PreparedBinding::Bind { + client, + table, + metadata, + } => ( + Some(state.bound_to(*table, (*metadata).clone())), + Some(client), + ), + PreparedBinding::Pend { client, name } => { + (Some(state.awaiting_table(client, name)), None) + }, + }; + + let rebinds = rebound.is_some(); + let mut next = rebound.unwrap_or_else(|| (*state).clone()); + next = next.with_config(view.config.clone()); + if let Some(overlay) = view.overlay { + next = next.with_overlay(overlay); + } + + if let Some(description) = view.description { + if let Some(bound) = next.bound() { + let mut metadata = (*bound.metadata).clone(); + if Self::record_description(&mut metadata, &view.effective, &description).is_ok() { + next = next.with_metadata(Rc::new(metadata)); + } } - self.borrow_mut().last_validated_expressions = Some(snap.effective.expressions.clone()); + next = next.with_description(Some((view.effective, description))); + } + + let next = rest(next); + let title_after = next.chrome.title.clone(); + let bound = next.bound().is_some(); + self.borrow_mut().swap(next); + let outgoing = if rebinds { + let mut data = self.borrow_mut(); + data.rendered = Rendered::Never; + data.view_sub.take() + } else { + None + }; + + if rebinds { + self.update_stats(ViewStats::default()); + } + + if (view.changed && view.announce) || rebinds { + self.notify_view_config_changed(); + } + + if title_before != title_after { + self.title_changed.emit(title_after); + } + + BindingEffects { + rebinds, + outgoing, + unloaded: rebinds && had_table, + loaded: rebinds && bound, + watch, + } + } + + /// Everything the metadata derives from a [`Description`] of `effective`: + /// the expression types, the types the `View` will have, and the window + /// columns. + fn record_description( + metadata: &mut SessionMetadata, + effective: &ViewConfig, + description: &Description, + ) -> ApiResult<()> { + metadata.update_expressions(&ExprValidationResult { + expression_schema: description.expression_schema.clone(), + errors: HashMap::new(), + expression_alias: effective.expressions.0.clone(), + })?; + + metadata.update_view_schema(&description.view_schema)?; + metadata.update_windows(&effective.windows)?; + Ok(()) + } + + /// Finish a committed rebind: dispose of the outgoing `View`, announce the + /// table change, and watch the incoming client for errors. + pub(crate) async fn finish_binding(&self, effects: BindingEffects) -> ApiResult<()> { + let BindingEffects { + outgoing, + unloaded, + loaded, + watch, + .. + } = effects; + + let deleted = outgoing.delete().await; + if unloaded { + self.table_unloaded.emit(true); + } + + if let Some(client) = watch { + self.watch_client(&client).await?; + } + + if loaded { + self.table_loaded.emit(()); } - Ok(ValidatedSnapshot(snap)) + deleted } /// Bind the engine `View` for a validated snapshot: SKIP @@ -1112,7 +1657,9 @@ impl Session { } } - let ConfigSnapshot { config, effective } = validated.0; + let ValidatedSnapshot { + snap: ConfigSnapshot { config, effective }, + } = validated; if self.borrow().is_paused { // A paused bind still RECONCILES the committed config (no `View` // is constructed — `view_created` stays silent). Without this @@ -1128,8 +1675,7 @@ impl Session { return Ok(unchanged_or_deferred(self)); } - let needs_schema = !self.metadata().has_view_schema(); - if !needs_schema { + { let bound = self.borrow().view_sub.as_ref().map(|x| x.build_config()); if let Some(bound) = bound { if *bound == *effective { @@ -1153,15 +1699,12 @@ impl Session { let table = self .borrow() - .table - .clone() + .table() + .cloned() .ok_or("`restore()` called before `load()`")?; let view_config = self.with_default_aggregates(&effective); let view = table.view(Some(view_config.into())).await?; - let view_schema = view.schema().await?; - self.metadata_mut().update_view_schema(&view_schema)?; - self.metadata_mut().update_windows(&effective.windows)?; let on_stats = Callback::from({ let this = self.clone(); move |stats| this.update_stats(stats) @@ -1192,6 +1735,13 @@ impl Session { /// The engine config for a `View` built from `effective`, per-column /// default aggregates filled in as a courtesy to the virtual server API. fn with_default_aggregates(&self, effective: &ViewConfig) -> ViewConfig { + Self::with_default_aggregates_of(&self.metadata(), effective) + } + + fn with_default_aggregates_of( + metadata: &SessionMetadata, + effective: &ViewConfig, + ) -> ViewConfig { let mut view_config = effective.clone(); for col in view_config .columns @@ -1200,8 +1750,7 @@ impl Session { .chain(view_config.sort.iter().map(|x| &x.0)) { if !view_config.aggregates.contains_key(col.as_str()) { - let agg = self - .metadata() + let agg = metadata .get_column_aggregates(col.as_str()) .and_then(|mut aggs| aggs.next()) .into_apierror(); @@ -1221,7 +1770,7 @@ impl Session { /// Build a caller-owned `View` from the effective config outside the /// render pipeline, with no subscription and no interaction with pause. pub async fn create_detached_view(&self) -> ApiResult { - let table = self.borrow().table.clone().ok_or("No `Table` set")?; + let table = self.borrow().table().cloned().ok_or("No `Table` set")?; let view_config = self.with_default_aggregates(&self.effective_view_config()); Ok(table.view(Some(view_config.into())).await?) } @@ -1242,7 +1791,7 @@ impl Session { Ok(JsValue::UNDEFINED) }); - self.borrow_mut().error = Some(TableErrorState( + let error = TableErrorState( err.clone(), Some(ReconnectCallback::new(move || { clone!(poll_loop); @@ -1251,7 +1800,10 @@ impl Session { Ok(()) }) })), - )); + ); + + let state = self.borrow().state.clone(); + self.borrow_mut().rendered = Rendered::Failed(state, error); if let Some(cb) = self.on_table_errored.borrow().as_ref() { cb.emit(()); @@ -1269,8 +1821,8 @@ impl Session { self.stats_changed.emit(()); } - fn all_columns(&self) -> Vec { - self.metadata() + fn columns_of(metadata: &SessionMetadata) -> Vec { + metadata .get_table_columns() .into_iter() .flatten() @@ -1283,6 +1835,8 @@ impl Session { /// session-related PubSub event fires. pub fn to_props(&self) -> SessionProps { let column_stats = PtrEqRc::new(self.column_stats.borrow().clone()); + let projected = self.projected(); + let title = self.get_title(); let data = self.borrow(); // Reuse memoized snapshots when the underlying value hasn't @@ -1292,15 +1846,15 @@ impl Session { // these `PtrEqRc`s will spuriously refire. let config = { let mut cached = self.cached_config.borrow_mut(); - if !matches!(&*cached, Some(c) if **c == data.config) { - *cached = Some(PtrEqRc::new(data.config.clone())); + if !matches!(&*cached, Some(c) if **c == *projected) { + *cached = Some(PtrEqRc::new((*projected).clone())); } cached.clone().unwrap() }; let metadata = { let mut cached = self.cached_metadata.borrow_mut(); - if !matches!(&*cached, Some(m) if **m == data.metadata) { - *cached = Some(PtrEqRc::new(data.metadata.clone())); + if !matches!(&*cached, Some(m) if **m == **data.metadata()) { + *cached = Some(PtrEqRc::new((**data.metadata()).clone())); } cached.clone().unwrap() }; @@ -1311,23 +1865,129 @@ impl Session { .stats .as_ref() .is_some_and(|s| s.num_table_cells.is_some()), - has_table: if data.table.is_some() { + has_table: if data.table().is_some() { Some(TableLoadState::Loaded) - } else if data.is_loading { + } else if self.0.queue.has_load() { Some(TableLoadState::Loading) - } else if data.pending_table.is_some() { + } else if data.pending_table().is_some() { Some(TableLoadState::Pending) } else { None }, - error: data.error.clone(), - title: data.title.clone(), + error: data.error().cloned(), + title, metadata, column_stats, } } } +/// A view-config change that has passed every check — names, normalization and +/// the server's `describe` — and only awaits [`Session::commit_view`]. +pub(crate) struct PreparedView { + binding: PreparedBinding, + + /// A new overlay to commit, when the op broadcasts one. + overlay: Option>>, + config: Rc, + effective: Rc, + description: Option>, + changed: bool, + + /// `false` for a projected UI edit, whose SUBMIT already announced it. + announce: bool, +} + +/// The plugin whose advice fills in what a restore's view config leaves unsaid +/// — computed against the table and config the restore LANDS on. +pub(crate) enum ViewDefaults<'a> { + /// The restore swaps to this plugin: default its columns and rollup mode. + Swap(&'a PluginStaticConfig), + + /// The restore stays on this plugin: re-enforce its rollup mode only. + Rollup(&'a PluginStaticConfig), + + /// The update is complete as given (a UI edit, whose control applied the + /// plugin's advice when it made it). + AsGiven, +} + +/// What a restore does to the panel's table binding — decided by its caller +/// from the restore's `table` and the panel it lands on, with the incoming +/// table already probed. +pub(crate) enum BindPlan { + /// The binding stands: bound, awaiting or unbound, as it is. + Keep, + + /// Bind `table`. + /// + /// Boxed: `Table` is a value handle several times the size of every other + /// variant here. + Bind { + client: Client, + table: Box, + reset: bool, + }, + + /// Await a host for `name`. + Pend { + client: Client, + name: String, + reset: bool, + }, +} + +enum PreparedBinding { + Keep, + Bind { + client: Client, + table: Box, + metadata: Rc, + }, + Pend { + client: Client, + name: String, + }, +} + +/// What [`Session::commit_view`] leaves for [`Session::finish_binding`]. +#[must_use] +pub(crate) struct BindingEffects { + rebinds: bool, + outgoing: Option, + unloaded: bool, + loaded: bool, + watch: Option, +} + +impl BindingEffects { + /// Whether the commit replaced the panel's table binding. + pub fn rebound(&self) -> bool { + self.rebinds + } + + /// Discard the effects of a commit that kept its binding (there are none). + pub fn forget(self) {} +} + +impl PreparedView { + /// Mark this as a projected UI edit: the UI has shown it since it was + /// submitted, so committing it announces nothing. + pub fn projected(mut self) -> Self { + self.announce = false; + self + } + + pub fn config(&self) -> &ViewConfig { + &self.config + } + + /// The type each column will have in the `View` this config builds. + pub fn view_schema(&self) -> Option<&HashMap> { + self.description.as_ref().map(|x| &x.view_schema) + } +} + /// One pipeline run's frozen input (invariant I2): the persisted /// [`ViewConfig`] and its EFFECTIVE companion (global-filter overlay /// appended), captured at the same synchronous instant inside the draw @@ -1338,10 +1998,11 @@ pub struct ConfigSnapshot { pub effective: Rc, } -/// Type-state token: proof this snapshot's expressions were validated by -/// [`Session::validate_snapshot`]. [`Session::bind_view`] accepts only this -/// token, so a `View` can never be built from an unvalidated snapshot. -pub struct ValidatedSnapshot(ConfigSnapshot); +/// Type-state token: proof this snapshot was validated by +/// [`Session::validate_snapshot`], carrying the server's [`Description`] of it. +pub struct ValidatedSnapshot { + snap: ConfigSnapshot, +} /// Type-state witness that a `View` is NEW for the plugin about to render /// it. diff --git a/rust/perspective-viewer/src/rust/session/metadata.rs b/rust/perspective-viewer/src/rust/session/metadata.rs index 942081ed04..80caea3ce7 100644 --- a/rust/perspective-viewer/src/rust/session/metadata.rs +++ b/rust/perspective-viewer/src/rust/session/metadata.rs @@ -13,6 +13,7 @@ use std::collections::{HashMap, HashSet}; use std::iter::IntoIterator; use std::ops::{Deref, DerefMut}; +use std::rc::Rc; use perspective_client::config::*; use perspective_js::apierror; @@ -56,7 +57,18 @@ impl DerefMut for SessionMetadata { } } -pub type MetadataRef<'a> = std::cell::Ref<'a, SessionMetadata>; +/// An OWNED read guard over a panel's [`SessionMetadata`]: a snapshot that +/// borrows nothing, so holding one can never make a later state write panic. +pub struct MetadataRef(pub(super) Rc); + +impl Deref for MetadataRef { + type Target = SessionMetadata; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + pub type MetadataMutRef<'a> = std::cell::RefMut<'a, SessionMetadata>; /// TODO the multiple `Option` types could probably be merged since they are @@ -290,6 +302,12 @@ impl SessionMetadata { self.as_ref().map(|meta| meta.edit_port) } + /// The type of a column of the `Table` itself — expression and window + /// columns, which belong to a config, excluded. + pub fn get_table_schema_type(&self, name: &str) -> Option { + self.as_ref()?.table_schema.get(name).copied() + } + /// Returns the type of a column name relative to the `Table`. Despite the /// name, `get_column_table_type()` also returns the `Table` type for /// Expressions, which despite living on the `View` still have a `table` diff --git a/rust/perspective-viewer/src/rust/session/op_queue.rs b/rust/perspective-viewer/src/rust/session/op_queue.rs new file mode 100644 index 0000000000..976a0724d0 --- /dev/null +++ b/rust/perspective-viewer/src/rust/session/op_queue.rs @@ -0,0 +1,393 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +//! The per-panel op queue: every writer of a panel's committed state is an +//! [`OpKind`] submitted here, and the queue's single drain runs them one at a +//! time in SUBMIT order. + +use std::cell::{Cell, RefCell}; +use std::collections::{BTreeSet, VecDeque}; + +use futures::channel::oneshot; +use futures::future::LocalBoxFuture; +use perspective_client::config::ViewConfigUpdate; +use perspective_js::utils::*; + +use crate::config::ColumnConfigFieldUpdate; +use crate::utils::InFlightGuard; + +/// What an op's step hands back to the drain. +pub enum StepOutcome { + /// The op is finished; its ticket resolves now. + Done, + + /// The op's WRITE is finished, and the drain may run the next op; its + /// ticket resolves when this render lands. + Render(LocalBoxFuture<'static, ApiResult<()>>), +} + +pub type StepFuture = LocalBoxFuture<'static, ApiResult>; +type Step = Box StepFuture>; + +/// The top-level config keys an op OVERWRITES. +pub type Fields = Option>; + +/// What a UI edit changes — enough to PROJECT it onto the committed state +/// before the drain commits it. +#[derive(Clone)] +pub enum EditDelta { + /// Boxed: `ViewConfigUpdate` is several times the size of every other + /// variant, and every queued op pays for it. + View(Box), + + /// One plugin-level settings field of the selected plugin. + PluginField(ColumnConfigFieldUpdate), + + /// One style field of one column, for the selected plugin. + ColumnField { + column: String, + update: ColumnConfigFieldUpdate, + }, + + /// Keys merged into the selected plugin's `plugin_config`. + PluginConfig(serde_json::Map), + Theme(Option), + Title(Option), +} + +pub enum OpKind { + /// A UI edit. + Edit { delta: EditDelta, fields: Fields }, + + /// A restore-family op. + Restore { fields: Fields }, + + /// The element's global filter, broadcast to this panel. + Overlay, + + /// A `load()`. + Load { table_known: bool }, +} + +impl OpKind { + fn fields(&self) -> &Fields { + match self { + OpKind::Edit { fields, .. } | OpKind::Restore { fields } => fields, + OpKind::Load { .. } | OpKind::Overlay => &None, + } + } + + /// Whether this LATER op makes the `earlier` one pointless: everything + /// `earlier` would write, this op overwrites. + fn covers(&self, earlier: &OpKind) -> bool { + match (self, earlier) { + (OpKind::Overlay, OpKind::Overlay) => true, + (OpKind::Overlay, _) | (_, OpKind::Overlay) => false, + (OpKind::Load { .. }, OpKind::Load { .. }) => true, + (OpKind::Load { table_known }, _) => *table_known, + (_, OpKind::Load { .. }) => false, + (later, earlier) => match (later.fields(), earlier.fields()) { + (Some(later), Some(earlier)) => earlier.is_subset(later), + _ => false, + }, + } + } +} + +/// A submitted op's handle: resolves exactly once, when the op (or, for a +/// superseded op, the op that superseded it) has finished AND rendered. +pub struct Ticket(oneshot::Receiver>); + +impl Ticket { + /// A ticket for an op that will never run (the panel is disposed). + pub fn settled(result: ApiResult<()>) -> Self { + let (sender, receiver) = oneshot::channel(); + let _ = sender.send(result); + Ticket(receiver) + } + + pub async fn settle(self) -> ApiResult<()> { + match self.0.await { + Ok(result) => result, + Err(_) => Err(ApiError::new("Cancelled")), + } + } +} + +/// What a running step can ask of the queue. +#[derive(Clone)] +pub struct OpCtx { + queue: std::rc::Rc, + kind_is_load: bool, +} + +impl OpCtx { + /// Whether a `load()` submitted since this op began makes its table binding + /// pointless: a `load()` abandons for ANY later `load()`, a restore's + /// rebind only for one already known to carry a `Table`. + pub fn is_superseded(&self) -> bool { + self.queue + .entries + .borrow() + .iter() + .any(|later| match later.kind { + OpKind::Load { table_known } => self.kind_is_load || table_known, + _ => false, + }) + } +} + +struct Entry { + kind: OpKind, + step: Step, + replies: Vec>>, + guards: Vec, +} + +impl Entry { + fn resolve(self, result: ApiResult<()>) { + resolve(self.replies, self.guards, result) + } +} + +fn resolve( + replies: Vec>>, + guards: Vec, + result: ApiResult<()>, +) { + for reply in replies { + let _ = reply.send(result.clone()); + } + + drop(guards); +} + +/// One unsettled op's accounts, released together when its ticket resolves — on +/// every path (rendered, rejected, superseded, disposed): the busy indicator, +/// and the queue's own count of ops still in flight. +struct Pending { + _busy: InFlightGuard, + queue: std::rc::Weak, +} + +impl Drop for Pending { + fn drop(&mut self) { + if let Some(queue) = self.queue.upgrade() { + queue.in_flight.set(queue.in_flight.get() - 1); + if queue.in_flight.get() == 0 { + for waiter in queue.idle_waiters.take() { + let _ = waiter.send(()); + } + } + } + } +} + +/// What happened to an entry the drain took off the queue without running. +pub enum Exit { + Superseded, + Rejected, +} + +#[derive(Default)] +pub struct OpQueue { + entries: RefCell>, + draining: Cell, + load_running: Cell, + + /// Ops whose tickets have not resolved — queued, running, or committed with + /// a detached render still to land. + in_flight: Cell, + idle_waiters: RefCell>>, +} + +impl OpQueue { + /// Append an op. + pub fn push( + self: &std::rc::Rc, + kind: OpKind, + guard: InFlightGuard, + step: impl FnOnce(OpCtx) -> StepFuture + 'static, + ) -> (Ticket, bool) { + let (sender, receiver) = oneshot::channel(); + self.in_flight.set(self.in_flight.get() + 1); + self.entries.borrow_mut().push_back(Entry { + kind, + step: Box::new(step), + replies: vec![sender], + guards: vec![Pending { + _busy: guard, + queue: std::rc::Rc::downgrade(self), + }], + }); + + (Ticket(receiver), !self.draining.replace(true)) + } + + /// The deltas of every pending [`OpKind::Edit`], in submit order. + pub fn pending_edits(&self) -> Vec { + self.entries + .borrow() + .iter() + .filter_map(|entry| match &entry.kind { + OpKind::Edit { delta, .. } => Some(delta.clone()), + _ => None, + }) + .collect() + } + + /// Resolves once every submitted op's ticket has — nothing queued, nothing + /// running, no committed render still to land. + pub async fn idle(&self) { + while self.in_flight.get() > 0 { + let (sender, receiver) = oneshot::channel(); + self.idle_waiters.borrow_mut().push(sender); + let _ = receiver.await; + } + } + + /// Whether a `load()` is queued or running. + pub fn has_load(&self) -> bool { + self.load_running.get() || self.has_queued_load() + } + + /// Whether a `load()` is queued behind the running op. + pub fn has_queued_load(&self) -> bool { + self.entries + .borrow() + .iter() + .any(|entry| matches!(entry.kind, OpKind::Load { .. })) + } + + /// Run every queued op, one at a time, in submit order. + pub async fn drain(self: std::rc::Rc, on_exit: impl Fn(Exit)) { + loop { + let Some(mut entry) = self.entries.borrow_mut().pop_front() else { + break; + }; + + let is_edit = matches!(entry.kind, OpKind::Edit { + delta: EditDelta::View(_), + .. + }); + let superseded = { + let mut entries = self.entries.borrow_mut(); + match entries + .iter_mut() + .find(|later| later.kind.covers(&entry.kind)) + { + Some(later) => { + later.replies.append(&mut entry.replies); + later.guards.append(&mut entry.guards); + true + }, + None => false, + } + }; + + if superseded { + if is_edit { + on_exit(Exit::Superseded); + } + + continue; + } + + let Entry { + kind, + step, + replies, + guards, + } = entry; + + let kind_is_load = matches!(kind, OpKind::Load { .. }); + self.load_running.set(kind_is_load); + let outcome = step(OpCtx { + queue: self.clone(), + kind_is_load, + }) + .await; + + self.load_running.set(false); + match outcome { + Ok(StepOutcome::Done) => resolve(replies, guards, Ok(())), + Ok(StepOutcome::Render(render)) => ApiFuture::spawn(async move { + resolve(replies, guards, render.await); + Ok(()) + }), + Err(error) => { + if is_edit { + tracing::warn!("Edit rejected: {}", error); + on_exit(Exit::Rejected); + } + + resolve(replies, guards, Err(error)); + }, + } + } + + self.draining.set(false); + } + + /// Reject everything still queued — the panel is gone. + pub fn reject_all(&self, error: ApiResult<()>) { + let entries = std::mem::take(&mut *self.entries.borrow_mut()); + for entry in entries { + entry.resolve(error.clone()); + } + } +} + +/// The top-level view-config keys `update` overwrites. +pub fn view_fields(update: &ViewConfigUpdate) -> BTreeSet<&'static str> { + let ViewConfigUpdate { + group_by, + split_by, + columns, + filter, + sort, + expressions, + windows, + aggregates, + group_by_depth, + filter_op, + group_rollup_mode, + split_rollup_mode, + } = update; + + let mut fields = BTreeSet::new(); + let mut set = |name: &'static str, present: bool| { + if present { + fields.insert(name); + } + }; + + set( + "group_by", + group_by.is_some() || group_rollup_mode.is_some(), + ); + set( + "group_rollup_mode", + group_by.is_some() || group_rollup_mode.is_some(), + ); + set("split_by", split_by.is_some()); + set("columns", columns.is_some()); + set("filter", filter.is_some()); + set("sort", sort.is_some()); + set("expressions", expressions.is_some()); + set("windows", windows.is_some()); + set("aggregates", aggregates.is_some()); + set("group_by_depth", group_by_depth.is_some()); + set("filter_op", filter_op.is_some()); + set("split_rollup_mode", split_rollup_mode.is_some()); + fields +} diff --git a/rust/perspective-viewer/src/rust/session/panel_state.rs b/rust/perspective-viewer/src/rust/session/panel_state.rs new file mode 100644 index 0000000000..78fa530344 --- /dev/null +++ b/rust/perspective-viewer/src/rust/session/panel_state.rs @@ -0,0 +1,463 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +use std::collections::HashMap; +use std::rc::Rc; + +use perspective_client::config::{ColumnType, Filter, ViewConfig}; +use perspective_client::{Client, Description, Table}; + +use super::TableErrorState; +use super::metadata::SessionMetadata; +use crate::config::PluginStaticConfig; +use crate::renderer::PluginScopedConfig; + +/// One panel's committed state, as ONE immutable value: a holder of an +/// `Rc` sees a consistent snapshot forever, and a change is a new +/// value swapped in whole. +#[derive(Clone, Default)] +pub struct PanelState { + pub chrome: Chrome, + + /// The selected plugin, or `None` until the first run selects one. + pub plugin: Option, + + /// Every plugin's own `plugin_config` / `columns_config`, keyed by plugin + /// name, so a swap away and back finds a plugin's config as it left it. + pub buckets: Rc>, + pub binding: Binding, + + /// The element's global filter as broadcast to this panel — the WHOLE set. + pub overlay: Rc>, +} + +/// One clause of the element-level global filter, with the type its column has +/// in the panel that BROADCAST it. +#[derive(Clone, Debug, PartialEq)] +pub struct OverlayClause { + pub filter: Filter, + pub column_type: Option, +} + +impl From for OverlayClause { + /// A clause with no broadcaster. + fn from(filter: Filter) -> Self { + Self { + filter, + column_type: None, + } + } +} + +/// The config a `View` is built from: `config`, plus each `overlay` clause the +/// listening table can honor — one naming a column the listener has, with the +/// broadcaster's type. +pub fn effective( + config: &ViewConfig, + overlay: &[OverlayClause], + type_of: &dyn Fn(&str) -> Option, +) -> (ViewConfig, Vec) { + let mut effective = config.clone(); + let mut skipped = Vec::new(); + for (idx, clause) in overlay.iter().enumerate() { + let applies = match (type_of(clause.filter.column()), clause.column_type) { + (Some(listener), Some(broadcaster)) => listener == broadcaster, + (Some(_), None) => true, + (None, _) => false, + }; + + if applies { + effective.filter.push(clause.filter.clone()); + } else { + skipped.push(idx); + } + } + + (effective, skipped) +} + +/// Panel presentation that is not a function of the data. +#[derive(Clone, Default)] +pub struct Chrome { + pub title: Option, + + /// This panel's theme name — CONCRETE, resolved once at creation from the + /// config's `theme`, else the host's, else the registry default. + pub theme: Option, +} + +/// A plugin selection: its index in the renderer's plugin store and the static +/// config that index resolved to. +#[derive(Clone)] +pub struct PluginRef { + pub idx: usize, + pub static_config: Rc, +} + +/// What this panel is bound to. +#[derive(Clone)] +pub enum Binding { + /// No table requested. + Unbound { + client: Option, + config: Rc, + }, + + /// A named table that no client hosts yet; `client` is the one asked. + Awaiting { + client: Client, + name: String, + config: Rc, + }, + + /// Bound to `table`. + Bound(Rc), + + /// The binding failed or its connection dropped. + Lost { + prior: Rc, + error: TableErrorState, + }, +} + +impl Default for Binding { + fn default() -> Self { + Self::Unbound { + client: None, + config: Rc::default(), + } + } +} + +#[derive(Clone)] +pub struct BoundState { + pub table: Table, + pub metadata: Rc, + pub config: Rc, + + /// The server's [`Description`] of the last validated effective config, + /// keyed by that config. + pub description: Option<(Rc, Rc)>, +} + +impl Binding { + /// This binding, seen through a [`Binding::Lost`] to what it was. + fn live(&self) -> &Binding { + match self { + Binding::Lost { prior, .. } => prior, + binding => binding, + } + } + + /// `f` applied to the live binding, preserving a `Lost` wrapper. + fn map_live(&self, f: impl FnOnce(&Binding) -> Binding) -> Binding { + match self { + Binding::Lost { prior, error } => Binding::Lost { + prior: Rc::new(f(prior)), + error: error.clone(), + }, + binding => f(binding), + } + } +} + +impl PanelState { + pub fn config(&self) -> &Rc { + match self.binding.live() { + Binding::Unbound { config, .. } | Binding::Awaiting { config, .. } => config, + Binding::Bound(bound) => &bound.config, + Binding::Lost { .. } => unreachable!("`Lost::prior` is never `Lost`"), + } + } + + /// The client this panel is bound through, awaits a table on, or will look + /// one up on. + pub fn client(&self) -> Option { + match self.binding.live() { + Binding::Unbound { client, .. } => client.clone(), + Binding::Awaiting { client, .. } => Some(client.clone()), + Binding::Bound(bound) => Some(bound.table.get_client()), + Binding::Lost { .. } => unreachable!("`Lost::prior` is never `Lost`"), + } + } + + pub fn bound(&self) -> Option<&Rc> { + match self.binding.live() { + Binding::Bound(bound) => Some(bound), + _ => None, + } + } + + pub fn awaiting(&self) -> Option<&str> { + match self.binding.live() { + Binding::Awaiting { name, .. } => Some(name), + _ => None, + } + } + + pub fn lost(&self) -> Option<&TableErrorState> { + match &self.binding { + Binding::Lost { error, .. } => Some(error), + _ => None, + } + } + + /// This state with its binding lost to `error`. + pub fn with_lost(&self, error: TableErrorState) -> Self { + Self { + binding: Binding::Lost { + prior: Rc::new(self.binding.live().clone()), + error, + }, + ..self.clone() + } + } + + /// This state with a lost binding recovered to what it was. + pub fn recovered(&self) -> Self { + Self { + binding: self.binding.live().clone(), + ..self.clone() + } + } + + /// This state with `config` in place of its view config. + pub fn with_config(&self, config: Rc) -> Self { + let binding = self.binding.map_live(|binding| match binding { + Binding::Unbound { client, .. } => Binding::Unbound { + client: client.clone(), + config, + }, + Binding::Awaiting { client, name, .. } => Binding::Awaiting { + client: client.clone(), + name: name.clone(), + config, + }, + Binding::Bound(bound) => Binding::Bound(Rc::new(BoundState { + config, + description: None, + ..(**bound).clone() + })), + Binding::Lost { .. } => unreachable!("`Lost::prior` is never `Lost`"), + }); + + Self { + binding, + ..self.clone() + } + } + + /// This state with nothing bound and nothing awaited (a lost binding is + /// replaced, not recovered); the client and config are kept. + pub fn unbound(&self) -> Self { + self.unbound_on(self.client()) + } + + /// This state with nothing bound, looking tables up on `client`. + pub fn unbound_on(&self, client: Option) -> Self { + Self { + binding: Binding::Unbound { + client, + config: self.config().clone(), + }, + ..self.clone() + } + } + + /// This state awaiting a host for `name` on `client`; the config is kept. + pub fn awaiting_table(&self, client: Client, name: String) -> Self { + Self { + binding: Binding::Awaiting { + client, + name, + config: self.config().clone(), + }, + ..self.clone() + } + } + + /// This state with `overlay` broadcast to it. + pub fn with_overlay(&self, overlay: Rc>) -> Self { + Self { + overlay, + ..self.with_description(None) + } + } + + /// This state bound to `table`; the config is kept and nothing has been + /// described against the new table yet. + pub fn bound_to(&self, table: Table, metadata: SessionMetadata) -> Self { + Self { + binding: Binding::Bound(Rc::new(BoundState { + table, + metadata: Rc::new(metadata), + config: self.config().clone(), + description: None, + })), + ..self.clone() + } + } + + /// This state with `metadata` in place of its table metadata; a no-op + /// unless bound. + pub fn with_metadata(&self, metadata: Rc) -> Self { + let binding = self.binding.map_live(|binding| match binding { + Binding::Bound(bound) => Binding::Bound(Rc::new(BoundState { + metadata, + ..(**bound).clone() + })), + binding => binding.clone(), + }); + + Self { + binding, + ..self.clone() + } + } + + /// This state with `description` recorded; a no-op unless bound. + pub fn with_description(&self, description: Option<(Rc, Rc)>) -> Self { + let binding = self.binding.map_live(|binding| match binding { + Binding::Bound(bound) => Binding::Bound(Rc::new(BoundState { + description, + ..(**bound).clone() + })), + binding => binding.clone(), + }); + + Self { + binding, + ..self.clone() + } + } + + pub fn with_title(&self, title: Option) -> Self { + Self { + chrome: Chrome { + title, + ..self.chrome.clone() + }, + ..self.clone() + } + } + + pub fn with_theme(&self, theme: Option) -> Self { + Self { + chrome: Chrome { + theme, + ..self.chrome.clone() + }, + ..self.clone() + } + } + + pub fn with_plugin(&self, plugin: PluginRef) -> Self { + Self { + plugin: Some(plugin), + ..self.clone() + } + } + + /// This state with no plugin selected and every bucket forgotten — a + /// deleted renderer's selection and buckets go with it. + pub fn without_plugins(&self) -> Self { + Self { + plugin: None, + buckets: Rc::default(), + ..self.clone() + } + } + + /// The named plugin's bucket, empty when it has never been written. + pub fn bucket(&self, name: &str) -> PluginScopedConfig { + self.buckets.get(name).cloned().unwrap_or_default() + } + + /// This state with the named plugin's bucket replaced. + pub fn with_bucket(&self, name: &str, bucket: PluginScopedConfig) -> Self { + let mut buckets = (*self.buckets).clone(); + buckets.insert(name.to_owned(), bucket); + Self { + buckets: Rc::new(buckets), + ..self.clone() + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn clause(column: &str, column_type: Option) -> OverlayClause { + OverlayClause { + filter: serde_json::from_value(serde_json::json!([column, "==", "x"])).unwrap(), + column_type, + } + } + + fn listener(name: &str) -> Option { + match name { + "region" => Some(ColumnType::String), + "sales" => Some(ColumnType::Float), + _ => None, + } + } + + #[test] + fn a_clause_applies_when_the_listener_has_its_column_and_type() { + let overlay = [clause("region", Some(ColumnType::String))]; + let (config, skipped) = effective(&ViewConfig::default(), &overlay, &listener); + assert_eq!(config.filter, vec![overlay[0].filter.clone()]); + assert!(skipped.is_empty()); + } + + #[test] + fn a_clause_is_skipped_when_the_listener_types_the_column_differently() { + let overlay = [clause("sales", Some(ColumnType::String))]; + let (config, skipped) = effective(&ViewConfig::default(), &overlay, &listener); + assert!(config.filter.is_empty()); + assert_eq!(skipped, vec![0]); + } + + #[test] + fn a_clause_is_skipped_when_the_listener_lacks_the_column() { + let overlay = [ + clause("nope", Some(ColumnType::String)), + clause("region", Some(ColumnType::String)), + ]; + + let (config, skipped) = effective(&ViewConfig::default(), &overlay, &listener); + assert_eq!(config.filter, vec![overlay[1].filter.clone()]); + assert_eq!(skipped, vec![0]); + } + + #[test] + fn an_untyped_clause_matches_on_name() { + let overlay = [clause("sales", None), clause("nope", None)]; + let (config, skipped) = effective(&ViewConfig::default(), &overlay, &listener); + assert_eq!(config.filter, vec![overlay[0].filter.clone()]); + assert_eq!(skipped, vec![1]); + } + + #[test] + fn the_overlay_follows_the_panel_s_own_filters() { + let own = clause("sales", None).filter; + let config = ViewConfig { + filter: vec![own.clone()], + ..ViewConfig::default() + }; + + let overlay = [clause("region", Some(ColumnType::String))]; + let (config, _) = effective(&config, &overlay, &listener); + assert_eq!(config.filter, vec![own, overlay[0].filter.clone()]); + } +} diff --git a/rust/perspective-viewer/src/rust/tasks/apply_global_filters.rs b/rust/perspective-viewer/src/rust/tasks/apply_global_filters.rs index ee0b5c2620..5034b5d5a9 100644 --- a/rust/perspective-viewer/src/rust/tasks/apply_global_filters.rs +++ b/rust/perspective-viewer/src/rust/tasks/apply_global_filters.rs @@ -10,50 +10,67 @@ // ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ // ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ -use perspective_client::config::ViewConfigUpdate; +use std::rc::Rc; -use super::apply_and_render; -use crate::session::Session; +use perspective_client::clone; +use perspective_js::utils::ApiFuture; + +use super::pipeline::{RunCommit, render_run}; +use super::transactional_restore::{commit_edit, prepare_overlay}; +use crate::session::{OpKind, OverlayClause, StepOutcome}; use crate::utils::spawn_owned; -use crate::workspace::{PanelId, Workspace}; +use crate::workspace::{Panel, PanelId, Workspace}; -/// Stamp `session`'s transient global-filter overlay from the -/// [`Workspace`]'s current set — EMPTY for master (filter-source) panels, -/// the full set for details. Master immunity is enforced here and only -/// here. Returns whether the overlay changed. -/// -/// Synchronous, so panel-creation paths can stamp BEFORE the panel's first -/// locked bind — the initial render then picks the overlay up via its -/// `ConfigSnapshot.effective`, with no second render and no unfiltered -/// first paint. -pub fn stamp_global_overlay(workspace: &Workspace, id: &PanelId, session: &Session) -> bool { - let filters = if workspace.is_master(id) { +/// The element's global filter as broadcast to panel `id` — EMPTY for master +/// (filter-source) panels, the full set for details. +pub(crate) fn overlay_for(workspace: &Workspace, id: &PanelId) -> Rc> { + Rc::new(if workspace.is_master(id) { Vec::new() } else { - workspace.global_filters() - }; + workspace.overlay() + }) +} + +/// Broadcast the [`Workspace`]'s current global filter to one panel, as an op +/// on ITS queue: the overlay is panel state, committed with the description of +/// the config it yields, like any other write. +pub fn broadcast_overlay(workspace: &Workspace, panel: &Panel) -> ApiFuture<()> { + let overlay = overlay_for(workspace, &panel.id); + clone!(panel.session, panel.renderer); + let ticket = panel.session.submit(OpKind::Overlay, move |_ctx| { + Box::pin(async move { + if *session.committed_overlay() == *overlay { + return Ok(StepOutcome::Done); + } + + let bound = session.get_table().is_some(); + let prepared = prepare_overlay(&session, &renderer, overlay).await?; + let committed = commit_edit(&session, &renderer, prepared); + Ok(if bound { + StepOutcome::Render(Box::pin(render_run( + session, + renderer, + RunCommit::Done(committed), + ))) + } else { + StepOutcome::Done + }) + }) + }); - session.set_global_filter(filters) + ApiFuture::new(ticket.settle()) } -/// Re-stamp every panel's overlay and re-render those whose overlay CHANGED -/// (details are usually non-active, so they need an explicit re-render -/// rather than relying on a subscription; unchanged panels aren't touched -/// at all). +/// Broadcast the global filter to every panel; panels whose overlay is +/// unchanged aren't touched at all. pub fn apply_global_filters(workspace: &Workspace) { - for pid in workspace.panel_ids() { - if let Some(panel) = workspace.panel(&pid) - && stamp_global_overlay(workspace, &pid, &panel.session) - { - let effect = workspace.effects().guard(); - let session = panel.session.clone(); - let renderer = panel.renderer.clone(); - spawn_owned("apply-global-filters", async move { - let _effect = effect; - apply_and_render(&session, &renderer, ViewConfigUpdate::default())?.await?; - Ok(()) - }); - } + for panel in workspace.panels() { + let effect = workspace.effects().guard(); + let task = broadcast_overlay(workspace, &panel); + spawn_owned("apply-global-filters", async move { + let _effect = effect; + task.await + }); } } diff --git a/rust/perspective-viewer/src/rust/tasks/auto_pause.rs b/rust/perspective-viewer/src/rust/tasks/auto_pause.rs index 7463505bfd..7c12985e1b 100644 --- a/rust/perspective-viewer/src/rust/tasks/auto_pause.rs +++ b/rust/perspective-viewer/src/rust/tasks/auto_pause.rs @@ -18,11 +18,12 @@ use wasm_bindgen::JsCast; use wasm_bindgen::prelude::*; use web_sys::*; +use super::transactional_restore::restore_in_place; use crate::config::ViewerConfigUpdate; use crate::js::*; use crate::presentation::Presentation; use crate::renderer::*; -use crate::session::Session; +use crate::session::{OpKind, Session, StepOutcome}; use crate::utils::*; use crate::workspace::Workspace; use crate::*; @@ -148,15 +149,25 @@ pub(crate) async fn set_panel_paused( ) -> ApiResult<()> { if visible { if session.set_pause(false) { - let result = super::restore_and_render( - session, - renderer, - presentation, - super::RunOrigin::Internal, - ViewerConfigUpdate::default(), - async move { Ok(()) }, - ) - .await; + let ticket = session.submit(OpKind::Restore { fields: None }, { + clone!(session, renderer, presentation); + move |_ctx| { + Box::pin(async move { + restore_in_place( + &session, + &renderer, + &presentation, + super::RunOrigin::Internal, + ViewerConfigUpdate::default(), + ) + .await?; + + Ok(StepOutcome::Done) + }) + } + }); + + let result = ticket.settle().await; if let Err(e) = result.ignore_view_delete() { session.set_run_error(e.clone()).await?; diff --git a/rust/perspective-viewer/src/rust/tasks/create_panel.rs b/rust/perspective-viewer/src/rust/tasks/create_panel.rs index c07767da6e..6e36f03551 100644 --- a/rust/perspective-viewer/src/rust/tasks/create_panel.rs +++ b/rust/perspective-viewer/src/rust/tasks/create_panel.rs @@ -97,7 +97,6 @@ pub(crate) async fn create_panel( Placement::Staged, ); - stamp_global_overlay(workspace, &id, &session); notify.emit(()); ApiFuture::spawn({ @@ -123,12 +122,12 @@ pub(crate) async fn create_panel( update, crate::tasks::RestoreErrors::Publish, missing, + None, ) .await; - // Promote on completion AND error — a failed restore's error state - // must become visible too. - if workspace.promote(&id) { + let discarded = discard_rejected(workspace, &id, &session, &result).await?; + if discarded || workspace.promote(&id) { notify.emit(()); } @@ -136,6 +135,26 @@ pub(crate) async fn create_panel( Ok(id) } +/// Remove the panel a rejected CREATING restore was for. +pub(crate) async fn discard_rejected( + workspace: &Workspace, + id: &PanelId, + session: &Session, + result: &ApiResult<()>, +) -> ApiResult { + if result.is_ok() || session.get_table().is_some() || session.pending_table().is_some() { + return Ok(false); + } + + match workspace.remove_panel(id) { + Some(panel) => { + eject_panel(panel, Disposal::Resolve).await?; + Ok(true) + }, + None => Ok(false), + } +} + /// Where [`create_panel_model`] registers the new panel model. pub(crate) enum Placement { /// Into the placed panel set at [`PanelPhase::Placed`] — a live, visible @@ -170,7 +189,7 @@ pub(crate) fn create_panel_model( placement: Placement, ) -> (PanelId, Session, Renderer, ViewerConfigUpdate) { let session = Session::new(); - let renderer = Renderer::new(elem); + let renderer = Renderer::new(elem, session.cell()); let id = id.unwrap_or_else(|| workspace.generate_id()); renderer.set_slot_name(id.as_str()); let subs = wire_panel_subs(elem, presentation, &session, &renderer); @@ -182,7 +201,7 @@ pub(crate) fn create_panel_model( } update.settings = OptionalUpdate::Missing; - renderer.set_theme(match &update.theme { + renderer.commit_theme(match &update.theme { OptionalUpdate::Update(theme) => Some(theme.clone()), _ => presentation.active_theme_name_sync(), }); @@ -213,7 +232,6 @@ pub(crate) fn place_reserved( has_table: bool, ) -> Option { let panel = workspace.claim_reserved(has_table)?; - stamp_global_overlay(workspace, &panel.id, &panel.session); notify.emit(()); Some(panel) } @@ -223,7 +241,7 @@ pub(crate) fn place_reserved( /// Shared by the root's `ClosePanel` handler and `restoreWorkspace`'s /// batch replacement of the pre-existing panel set. pub(crate) fn eject_panel(panel: Panel, disposal: Disposal) -> ApiFuture<()> { - panel.session.mark_disposed(disposal); + panel.session.dispose(disposal); ApiFuture::new(async move { panel.renderer.dispose().await?; let was_errored = panel.session.is_errored(); diff --git a/rust/perspective-viewer/src/rust/tasks/mod.rs b/rust/perspective-viewer/src/rust/tasks/mod.rs index 05b543740b..07f9d47285 100644 --- a/rust/perspective-viewer/src/rust/tasks/mod.rs +++ b/rust/perspective-viewer/src/rust/tasks/mod.rs @@ -33,13 +33,13 @@ mod pipeline; mod presize_panels; mod reset_all; mod resize_observer; -mod restore_and_render; mod restore_panel; mod send_column_config; mod send_plugin_config; mod set_edit_mode; mod sync_update_panels; mod table_lifecycle; +mod transactional_restore; mod update_theme; mod validate_expression; @@ -65,12 +65,12 @@ pub use self::pipeline::*; pub use self::presize_panels::*; pub use self::reset_all::*; pub use self::resize_observer::*; -pub use self::restore_and_render::*; pub(crate) use self::restore_panel::*; pub use self::send_column_config::*; pub use self::send_plugin_config::*; pub use self::set_edit_mode::*; pub(crate) use self::sync_update_panels::*; pub(crate) use self::table_lifecycle::*; +pub(crate) use self::transactional_restore::{commit_and_render_locked, prepare}; pub use self::update_theme::*; pub use self::validate_expression::*; diff --git a/rust/perspective-viewer/src/rust/tasks/pipeline.rs b/rust/perspective-viewer/src/rust/tasks/pipeline.rs index 52f30e98cd..a463882c00 100644 --- a/rust/perspective-viewer/src/rust/tasks/pipeline.rs +++ b/rust/perspective-viewer/src/rust/tasks/pipeline.rs @@ -12,17 +12,18 @@ use std::rc::Rc; -use futures::future::LocalBoxFuture; use perspective_client::config::ViewConfigUpdate; use perspective_client::{View, clone}; use perspective_js::utils::*; use wasm_bindgen::JsValue; use yew::prelude::*; -use crate::config::{ColumnConfigUpdate, PluginConfigUpdate}; +use super::transactional_restore::{commit_edit, prepare_edit}; use crate::presentation::Presentation; use crate::renderer::{RenderContext, Renderer}; -use crate::session::{BindDisposition, Disposal, Session}; +use crate::session::{ + BindDisposition, BindingEffects, Disposal, EditDelta, OpKind, Session, StepOutcome, view_fields, +}; use crate::utils::RenderGuard; /// Snapshot → validate → bind → cache + pin the [`RenderContext`]. The core @@ -35,15 +36,6 @@ pub async fn bind_snapshot( session: &Session, renderer: &Renderer, ) -> ApiResult<(BindDisposition, Option)> { - // A `load()` is mid-classification (see [`Session::pending_load`]): DEFER - // rather than bind the incoming config against the still-bound outgoing - // table. Holds the last painted frame; the `load()` run performs the one - // reconciling bind once its payload resolves — the single visible - // transition, correct in whichever future (Table/Client) materializes. - if session.has_pending_load() { - return Ok((BindDisposition::Deferred, None)); - } - let snap = session.snapshot(guard); let validated = session.validate_snapshot(guard, snap).await?; let disposition = session.bind_view(guard, validated).await?; @@ -132,34 +124,76 @@ pub fn apply_and_render( } /// [`apply_and_render`] plus a plugin selection (resolved via -/// [`Renderer::resolve_plugin_update`]). The plugin swap is committed inside -/// the locked run, atomically with the view rebind — never staged on the -/// `Renderer` where a concurrent draw could observe it. +/// [`Renderer::resolve_plugin_update`]), committed in the SAME swap as the +/// view config — in the op's step, never by the detached render. pub fn update_plugin_and_render( session: &Session, renderer: &Renderer, update: ViewConfigUpdate, plugin_idx: Option, ) -> ApiResult> { - session.commit_view_config(update)?; - // Spinner accounting (RAII): created only when the commit succeeded, - // moved into the run future, settled on every exit path by `Drop`. - let run_token = session.begin_config_run(); - clone!(session, renderer); - Ok(ApiFuture::new(async move { - let _run_token = run_token; - render_run(session, renderer, plugin_idx).await - })) + session.check_edit(&update)?; + let fields = plugin_idx.is_none().then(|| view_fields(&update)); + let kind = OpKind::Edit { + delta: EditDelta::View(Box::new(update.clone())), + fields, + }; + + let ticket = session.submit(kind, { + clone!(session, renderer); + move |_ctx| { + Box::pin(async move { + let prepared = prepare_edit(&session, &renderer, update, plugin_idx).await?; + let committed = commit_edit(&session, &renderer, prepared); + Ok(StepOutcome::Render(Box::pin(render_run( + session, + renderer, + RunCommit::Done(committed), + )))) + }) + } + }); + + Ok(ApiFuture::new(ticket.settle())) } /// Re-render from the current commit without applying an update. pub fn just_render(session: &Session, renderer: &Renderer) -> ApiResult> { clone!(session, renderer); Ok(ApiFuture::new(async move { - render_run(session, renderer, None).await + render_run(session, renderer, RunCommit::None).await })) } +/// What a transactional restore committed (see +/// [`super::transactional_restore`]): the run has to bring the plugin element +/// and the paint in line with it. +pub(crate) struct Committed { + /// A plugin selection was committed; its element must be activated. + pub activate: bool, + + /// ...and it REPLACED an earlier selection (a first selection is not a + /// swap, and does not force a `plugin.restore`). + pub plugin_swapped: bool, + pub plugin_config_changed: bool, + pub columns_config_changed: bool, +} + +/// The commit a locked run renders. +pub(crate) enum RunCommit { + /// Whatever is committed (a repaint, an activation, a resize). + None, + + /// A UI edit its op's step ALREADY committed; the plugin receives the final + /// buckets in ONE `plugin.restore`. + Done(Committed), + + /// A restore's commit, run first thing under the lock — a restore may + /// REBIND, and under the lock no render of the outgoing state can bind a + /// `View` over the incoming table. + Deferred(Box (Committed, BindingEffects)>), +} + /// Everything that varies between locked render runs, consumed by /// [`locked_run`] — the ONE lock-body composition. `Default` is the /// host-internal UI-commit run: `Internal` origin, no plugin swap, no @@ -168,21 +202,8 @@ pub(crate) struct RunSpec { /// Who initiated this run (see [`RunOrigin`]). pub origin: RunOrigin, - /// A plugin swap resolved by [`Renderer::resolve_plugin_update`], - /// committed inside the run atomically with the rebind. - pub plugin_idx: Option, - - /// Plugin-level bucket update, applied inside the run after the bind - /// (so strip-on-write sees fresh schemas). - pub plugin_config: PluginConfigUpdate, - - /// Per-column bucket update; same timing as `plugin_config`. - pub columns_config: ColumnConfigUpdate, - - /// Pre-bind hook awaited inside the lock (table binding, resets) — - /// BEFORE the error guard, so a task that recovers the session (e.g. - /// `restorePanel`'s errored-recovery reset) unblocks its own run. - pub task: Option>>, + /// What this run renders — see [`RunCommit`]. + pub commit: RunCommit, /// When set, plugin dispatch is skipped while the host element is not /// visible (`Presentation::is_visible`) — the `restore` family's gate @@ -196,10 +217,7 @@ impl Default for RunSpec { fn default() -> Self { Self { origin: RunOrigin::Internal, - plugin_idx: None, - plugin_config: PluginConfigUpdate::Missing, - columns_config: ColumnConfigUpdate::Missing, - task: None, + commit: RunCommit::None, presentation: None, } } @@ -208,7 +226,7 @@ impl Default for RunSpec { /// One locked, witnessed, snapshot-consuming render run — the SINGLE lock /// body shared by every config-driven run (`apply_and_render` &co. via /// [`render_run`]'s tail, the `restore` family via -/// [`super::restore_and_render`]): eager mount → pre-bind `task` → +/// [`super::transactional_restore`]): the restore's commit → eager mount → /// plugin-swap commit → theme stamp → error guard → [`bind_snapshot`] /// (gated on a bound `Table`, else `Deferred` — the config is already /// committed, so the eventual `load()` run binds from it) → bucket updates @@ -226,7 +244,19 @@ pub(crate) async fn locked_run( clone!(session, renderer); renderer .clone() - .render_task(|guard| async move { + .render_task(|guard| run_locked(guard, session, renderer, spec)) + .await +} + +/// [`locked_run`]'s body, for a caller that already holds the draw lock. +pub(crate) async fn run_locked( + guard: RenderGuard, + session: Session, + renderer: Renderer, + spec: RunSpec, +) -> ApiResult<()> { + { + { if let Some(disposal) = session.disposal() { return match (disposal, spec.origin) { (Disposal::Reject, RunOrigin::Public) => Err(ApiError::new("Panel disposed")), @@ -234,60 +264,87 @@ pub(crate) async fn locked_run( }; } - renderer.mount_active_plugin()?; - if let Some(task) = spec.task { - task.await?; + let committed = match spec.commit { + RunCommit::Done(committed) => Some(committed), + RunCommit::Deferred(commit) => { + let (committed, binding) = commit(); + let rebound = binding.rebound(); + session.finish_binding(binding).await?; + if rebound + && session.get_table().is_none() + && let Some(plugin) = renderer.active_plugin() + { + plugin.clear().await?; + } + + Some(committed) + }, + RunCommit::None => None, + }; + + if !committed.as_ref().is_some_and(|c| c.activate) { + renderer.mount_active_plugin()?; } - let plugin_swapped = renderer.commit_plugin(spec.plugin_idx)?; + let plugin_swapped = match &committed { + Some(committed) => { + if committed.activate { + renderer.activate_committed_plugin()?; + } + + committed.plugin_swapped + }, + None => { + renderer.ensure_plugin_selected()?; + false + }, + }; let plugin = renderer.active_plugin().ok_or("No Plugin")?; renderer.stamp_theme(Some(&plugin)); - if let Some(error) = session.get_error() { + if let Some(error) = session.blocking_error() { return match spec.origin { RunOrigin::Public => Err(error), RunOrigin::Internal => Ok(()), }; } + session.set_rendered(false); + let (disposition, _pin) = if session.get_table().is_some() { bind_snapshot(&guard, &session, &renderer).await? } else { (BindDisposition::Deferred, None) }; - let view_config_snapshot = session.get_view_config().clone(); - let plugin_config_changed = - renderer.update_plugin_config(&view_config_snapshot, spec.plugin_config)?; - let plugin_config_snapshot = renderer.get_plugin_config(); - let plugin_update = - JsValue::from_serde_ext(&plugin_config_snapshot).unwrap_or(JsValue::NULL); - - let plugin_restored = plugin_config_changed || plugin_swapped; - if plugin_restored { - let columns_config = renderer - .all_columns_configs_materialized(&view_config_snapshot, &session) - .await; - plugin.restore(&plugin_update, Some(&columns_config))?; - } + let view_config_snapshot = session.committed_view_config().clone(); + let changed = match committed { + Some(committed) => { + let changed = + committed.plugin_config_changed || committed.columns_config_changed; - let columns_config_changed = renderer.update_columns_configs( - &view_config_snapshot, - &session, - spec.columns_config, - )?; - - let changed = plugin_config_changed || columns_config_changed; - if columns_config_changed { - let columns_config = renderer - .all_columns_configs_materialized(&view_config_snapshot, &session) - .await; - plugin.restore(&plugin_update, Some(&columns_config))?; - renderer.columns_config_changed.emit(columns_config); - } + if changed || plugin_swapped { + let plugin_config = renderer.committed_plugin_config(); + let plugin_update = + JsValue::from_serde_ext(&plugin_config).unwrap_or(JsValue::NULL); - if plugin_config_changed { - renderer.plugin_config_changed.emit(plugin_config_snapshot); - } + let columns_config = renderer + .all_columns_configs_materialized(&view_config_snapshot, &session) + .await; + + plugin.restore(&plugin_update, Some(&columns_config))?; + if committed.columns_config_changed { + renderer.columns_config_changed.emit(columns_config); + } + + if committed.plugin_config_changed { + renderer.plugin_config_changed.emit(plugin_config); + } + } + + changed + }, + None => false, + }; if spec .presentation @@ -298,22 +355,23 @@ pub(crate) async fn locked_run( dispatch_bound(&guard, &renderer, disposition, changed, spec.origin).await?; } + session.set_rendered(true); Ok(()) - }) - .await + } + } } /// [`locked_run`]'s host-internal tail: a failed RUN sets error state (with /// the reset-reconnect affordance); the committed config is NOT rolled back /// (I4). Cancellation by a superseding run ("View already deleted") is not /// a failure. -async fn render_run( +pub(super) async fn render_run( session: Session, renderer: Renderer, - plugin_idx: Option, + commit: RunCommit, ) -> ApiResult<()> { let spec = RunSpec { - plugin_idx, + commit, ..RunSpec::default() }; diff --git a/rust/perspective-viewer/src/rust/tasks/reset_all.rs b/rust/perspective-viewer/src/rust/tasks/reset_all.rs index 0564445633..5714dd2dff 100644 --- a/rust/perspective-viewer/src/rust/tasks/reset_all.rs +++ b/rust/perspective-viewer/src/rust/tasks/reset_all.rs @@ -11,20 +11,20 @@ // ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ //! Cross-engine reset orchestration: reset session config, optionally clear -//! presentation columns config / theme, then delegate to `restore_and_render` -//! to switch back to the default plugin and redraw. +//! presentation columns config / theme, then delegate to a transactional +//! restore to switch back to the default plugin and redraw. use perspective_client::clone; -use perspective_js::utils::ApiFuture; +use perspective_js::utils::{ApiFuture, ApiResult}; use super::pipeline::RunOrigin; -use super::restore_and_render; +use super::transactional_restore::restore_in_place; use crate::config::{ ColumnConfigUpdate, OptionalUpdate, PluginConfigUpdate, PluginUpdate, ViewerConfigUpdate, }; use crate::presentation::Presentation; use crate::renderer::Renderer; -use crate::session::{ResetOptions, Session}; +use crate::session::{OpKind, ResetOptions, Session, StepOutcome}; /// Reset the viewer's `ViewerConfig` to the default. /// @@ -37,7 +37,7 @@ use crate::session::{ResetOptions, Session}; /// futures like this one) and error, rather than this task spawning unowned /// work. /// -/// Delegates plugin selection + draw to [`restore_and_render`], whose +/// Delegates plugin selection + draw to [`restore_in_place`], whose /// two-pass restore guarantees the default plugin sees materialized /// `columns_config` / `plugin_config` on its first draw — fixing a race /// where the raw post-reset bucket would reach the plugin before @@ -49,8 +49,27 @@ pub fn reset_all( all: bool, ) -> ApiFuture<()> { presentation.set_open_column_settings(None); - clone!(session, renderer, presentation); - ApiFuture::new(async move { + let ticket = session.submit(OpKind::Restore { fields: None }, { + clone!(session, renderer, presentation); + move |_ctx| { + Box::pin(async move { + reset_all_step(&session, &renderer, &presentation, all).await?; + Ok(StepOutcome::Done) + }) + } + }); + + ApiFuture::new(ticket.settle()) +} + +/// The body of [`reset_all`], run as one op on the panel's queue. +async fn reset_all_step( + session: &Session, + renderer: &Renderer, + presentation: &Presentation, + all: bool, +) -> ApiResult<()> { + { session .reset(ResetOptions { config: true, @@ -65,10 +84,10 @@ pub fn reset_all( // `reset_theme` only resets the host, which an explicitly-themed // panel would otherwise keep overriding. presentation.reset_theme().await?; - renderer.set_theme(presentation.get_default_theme_name().await); + renderer.commit_theme(presentation.get_default_theme_name().await); } - // For `all = true`, route the bucket clears through `restore_and_render`'s + // For `all = true`, route the bucket clears through the restore's // `update_*` paths as `SetDefault`. This guarantees the materialized // restore fires even when the user is already on the default plugin // (no plugin_swap signal), since `SetDefault` reports the bucket as @@ -93,16 +112,8 @@ pub fn reset_all( // `reset()` is a public element API — `Public` keeps its repaint // affordance even on an already-default config. - restore_and_render( - &session, - &renderer, - &presentation, - RunOrigin::Public, - update, - async { Ok(()) }, - ) - .await?; + restore_in_place(session, renderer, presentation, RunOrigin::Public, update).await?; renderer.reset_changed.emit(()); Ok(()) - }) + } } diff --git a/rust/perspective-viewer/src/rust/tasks/restore_and_render.rs b/rust/perspective-viewer/src/rust/tasks/restore_and_render.rs deleted file mode 100644 index 09169fd125..0000000000 --- a/rust/perspective-viewer/src/rust/tasks/restore_and_render.rs +++ /dev/null @@ -1,107 +0,0 @@ -// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ -// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ -// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ -// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ -// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ -// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ -// ┃ Copyright (c) 2017, the Perspective Authors. ┃ -// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ -// ┃ This file is part of the Perspective library, distributed under the terms ┃ -// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ -// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ - -use futures::Future; -use perspective_client::clone; - -use super::pipeline::{RunOrigin, RunSpec, locked_run}; -use super::update_theme::seed_panel_theme; -use crate::config::{OptionalUpdate, ViewerConfigUpdate}; -use crate::presentation::Presentation; -use crate::renderer::Renderer; -use crate::session::Session; -use crate::*; - -/// Apply a full [`ViewerConfigUpdate`] (theme, title, plugin selection, -/// plugin config, columns config, view config) to the engines and re-draw. -/// Returns an [`ApiFuture<()>`] which resolves when the draw completes. -pub fn restore_and_render( - session: &Session, - renderer: &Renderer, - presentation: &Presentation, - origin: RunOrigin, - ViewerConfigUpdate { - plugin, - plugin_config, - columns_config, - settings, - theme: theme_name, - title, - mut view_config, - .. - }: ViewerConfigUpdate, - task: impl Future> + 'static, -) -> ApiFuture<()> { - clone!(session, renderer, presentation); - ApiFuture::new(async move { - if let OptionalUpdate::Update(x) = settings { - presentation.set_settings_attribute(x); - presentation.set_settings_before_open(x); - } - - if let OptionalUpdate::Update(title) = title { - session.set_title(Some(title)); - } else if matches!(title, OptionalUpdate::SetDefault) { - session.set_title(None); - } - - match theme_name { - OptionalUpdate::SetDefault => { - let current_name = presentation.get_selected_theme_name().await; - if current_name.is_some() { - presentation.set_theme_name(None).await?; - } - }, - OptionalUpdate::Update(x) => { - presentation.set_theme_name(Some(&x)).await?; - }, - _ => {}, - }; - - let resolved_plugin = renderer.resolve_plugin_update(&plugin); - if let Some((_, metadata)) = &resolved_plugin { - session.set_update_column_defaults(&mut view_config, metadata); - } else { - let metadata = if renderer.active_plugin().is_none() { - renderer - .resolve_plugin_update(&OptionalUpdate::SetDefault) - .map(|(_, metadata)| metadata) - .unwrap_or_else(|| renderer.metadata()) - } else { - renderer.metadata() - }; - - session.set_update_rollup_defaults(&mut view_config, &metadata); - } - - let plugin_idx = resolved_plugin.map(|(idx, _)| idx); - session.commit_view_config(view_config)?; - let _run_token = session.begin_config_run(); - seed_panel_theme(&presentation, &renderer).await; - locked_run(&session, &renderer, RunSpec { - origin, - plugin_idx, - plugin_config, - columns_config, - task: Some(Box::pin(task)), - presentation: Some(presentation.clone()), - }) - .await?; - - if renderer.needs_restyle() { - renderer.restyle_all().await?; - } - - presentation.publish_theme_config().await?; - Ok(()) - }) -} diff --git a/rust/perspective-viewer/src/rust/tasks/restore_panel.rs b/rust/perspective-viewer/src/rust/tasks/restore_panel.rs index e74135f180..28c1850885 100644 --- a/rust/perspective-viewer/src/rust/tasks/restore_panel.rs +++ b/rust/perspective-viewer/src/rust/tasks/restore_panel.rs @@ -10,6 +10,8 @@ // ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ // ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ +use std::collections::BTreeSet; + use perspective_client::clone; use perspective_client::utils::PerspectiveResultExt; @@ -17,10 +19,11 @@ use crate::config::*; use crate::presentation::Presentation; use crate::renderer::Renderer; use crate::session::{ - LoadGuard, MissingTable, ResetOptions, Session, TableIntermediateState, probe_table, + BindPlan, MissingTable, OpCtx, OpKind, Session, StepOutcome, probe_table, view_fields, }; +use crate::tasks::transactional_restore::{Outcome, commit_and_render, prepare}; use crate::tasks::*; -use crate::workspace::Workspace; +use crate::workspace::{PanelId, Workspace}; use crate::*; /// How a [`restore_panel`] call reached the pipeline — the only two genuine @@ -30,7 +33,7 @@ pub(crate) enum RestoreMode { Fresh, } -/// Where a failed restore's error goes. +/// Where the error of a restore that COMMITTED and then failed to render goes. #[derive(Clone, Copy)] pub(crate) enum RestoreErrors { // Raise errors in the UI. @@ -64,77 +67,116 @@ async fn probe( Ok((client, table)) } -/// Bind an unbound or same-named session to `name`. -pub(crate) async fn bind_table_task( - session: &Session, - workspace: &Workspace, - name: String, - missing: MissingTable, -) -> ApiResult<()> { - if session - .get_table() - .is_some_and(|t| t.get_name() == name.as_str()) - { - return Ok(()); - } - - let (client, table) = probe(session, workspace, &name, missing).await?; - session.set_client(client); - match table { - Some(table) => session.bind_table(table).await?, - None => session.pend_table(name).await?, - } +/// What a restore naming `table` does to the panel's binding. +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum Rebind { + /// A caller's restore: a table that REPLACES what the panel showed (or + /// recovers an errored panel) starts from a default config. + Replace, - session.commit_table_defaults(); - Ok(()) + /// The table lifecycle completing a pending bind: the committed config is + /// the intent the bind exists to honor. + Complete, } -/// Rebind a session to `name`, probing the incoming table before the -/// outgoing binding is dropped and replaying `load`'s journal over the -/// incoming table's defaults. -async fn rebind_table_task( +/// Decide a restore's [`BindPlan`], probing the incoming table — without +/// touching the session's own binding. +async fn plan_binding( + ctx: &OpCtx, session: &Session, - renderer: &Renderer, workspace: &Workspace, - name: String, + table: &TableUpdate, + fresh: bool, + rebind: Rebind, missing: MissingTable, - load: &LoadGuard, -) -> ApiResult<()> { - let probed = probe(session, workspace, &name, missing).await; - let journal = load.claim(); - let (client, table) = probed?; - let Some(journal) = journal else { - return Ok(()); +) -> ApiResult> { + let OptionalUpdate::Update(name) = table else { + return Ok(Some(BindPlan::Keep)); }; - session - .reset(ResetOptions { - config: true, - expressions: true, - stats: true, - table: Some(TableIntermediateState::Reloaded), - }) - .await?; - - session.set_client(client); - match table { - Some(table) => session.bind_table(table).await?, - None => session.pend_table(name).await?, + let same = session + .get_table() + .is_some_and(|t| t.get_name() == name.as_str()); + + if same && !session.is_errored() { + return Ok(Some(BindPlan::Keep)); } - session.commit_table_defaults(); - for delta in journal { - session.commit_view_config(delta)?; + let (client, table) = probe(session, workspace, name, missing).await?; + if !fresh && ctx.is_superseded() { + return Ok(None); } - session.commit_table_defaults(); - if session.get_table().is_none() - && let Some(plugin) = renderer.active_plugin() + if table.is_none() + && rebind == Rebind::Complete + && session.pending_table().as_deref() == Some(name.as_str()) { - plugin.clear().await?; + return Ok(None); } - Ok(()) + let reset = !fresh && rebind == Rebind::Replace; + Ok(Some(match table { + Some(table) => BindPlan::Bind { + client, + table: Box::new(table), + reset, + }, + None => BindPlan::Pend { + client, + name: name.clone(), + reset, + }, + })) +} + +/// The top-level config keys `update` OVERWRITES, or `None` when it cannot be +/// superseded: `plugin_config` / `columns_config` updates MERGE into their +/// buckets, and a fresh panel's restore is what creates it. +fn restore_fields( + update: &ViewerConfigUpdate, + mode: &RestoreMode, +) -> Option> { + let ViewerConfigUpdate { + version, + plugin, + plugin_config, + columns_config, + settings, + theme, + title, + table, + view_config, + } = update; + + let _ = version; + if matches!(mode, RestoreMode::Fresh) + || matches!(plugin_config, OptionalUpdate::Update(_)) + || matches!(columns_config, OptionalUpdate::Update(_)) + { + return None; + } + + let mut fields = view_fields(view_config); + let mut set = |name: &'static str, present: bool| { + if present { + fields.insert(name); + } + }; + + set("plugin", !matches!(plugin, OptionalUpdate::Missing)); + set( + "plugin_config", + !matches!(plugin_config, OptionalUpdate::Missing), + ); + set( + "columns_config", + !matches!(columns_config, OptionalUpdate::Missing), + ); + set("settings", !matches!(settings, OptionalUpdate::Missing)); + set("theme", !matches!(theme, OptionalUpdate::Missing)); + set("title", !matches!(title, OptionalUpdate::Missing)); + set("table", !matches!(table, OptionalUpdate::Missing)); + Some(fields) } /// Apply a [`ViewerConfigUpdate`] to a single panel and re-draw — the one @@ -147,110 +189,105 @@ pub(crate) async fn restore_panel( presentation: &Presentation, workspace: &Workspace, mode: RestoreMode, - mut update: ViewerConfigUpdate, + update: ViewerConfigUpdate, + errors: RestoreErrors, + missing: MissingTable, + preamble: Option>>, +) -> ApiResult<()> { + let fields = restore_fields(&update, &mode); + let ticket = session.submit(OpKind::Restore { fields }, { + clone!(session, renderer, presentation, workspace); + move |ctx| { + Box::pin(async move { + if let Some(preamble) = preamble { + preamble.await?; + } + + restore_panel_step( + &ctx, + &session, + &renderer, + &presentation, + &workspace, + mode, + Rebind::Replace, + RunOrigin::Public, + update, + errors, + missing, + ) + .await?; + + Ok(StepOutcome::Done) + }) + } + }); + + ticket.settle().await +} + +/// The body of [`restore_panel`], to be called only from a running op's step. +#[allow(clippy::too_many_arguments)] +pub(crate) async fn restore_panel_step( + ctx: &OpCtx, + session: &Session, + renderer: &Renderer, + presentation: &Presentation, + workspace: &Workspace, + mode: RestoreMode, + rebind: Rebind, + origin: RunOrigin, + update: ViewerConfigUpdate, errors: RestoreErrors, missing: MissingTable, ) -> ApiResult<()> { let active = matches!(mode, RestoreMode::Existing { active: true }); let fresh = matches!(mode, RestoreMode::Fresh); - match &update.theme { - OptionalUpdate::Update(theme) => renderer.set_theme_stamped(Some(theme.clone())), - // `SetDefault` resolves to a CONCRETE registry default here, rather - // than clearing the panel's theme — nothing downstream re-resolves. - OptionalUpdate::SetDefault => { - renderer.set_theme_stamped(presentation.get_default_theme_name().await) - }, - OptionalUpdate::Missing => {}, - } - if !active { - update.theme = OptionalUpdate::Missing; - } + renderer.check_plugin_update(&update.plugin)?; + let Some(plan) = plan_binding( + ctx, + session, + workspace, + &update.table, + fresh, + rebind, + missing, + ) + .await? + else { + return Ok(()); + }; if !fresh { tracing::info!("Restoring {update}"); } - // NOTE: `update.settings` is deliberately NOT applied here. It is - // element-level chrome rather than panel state, so `restore()` — the - // only caller that can carry it — applies it before dispatching, and - // this pipeline stays per-panel. Applying it here reached only the - // `Existing { active: true }` mode, which is why a freshly created - // panel silently ignored it. - - // Under `Suppress` the restore is TRANSACTIONAL, so snapshot the config - // it is about to overwrite — see the failure tail below for why. - let rollback = - matches!(errors, RestoreErrors::Suppress).then(|| session.get_view_config().clone()); - - let binding_before = ( - session.get_table().map(|t| t.get_name().to_owned()), - session.pending_table(), - ); - - let load = match &update.table { - OptionalUpdate::Update(name) - if session.is_errored() - || (!fresh - && session - .get_table() - .map(|t| t.get_name() != name.as_str()) - .unwrap_or(true)) => - { - Some(session.begin_pending_load()) - }, - _ => None, - }; + let overlay = renderer + .slot_name() + .map(|id| overlay_for(workspace, &PanelId::from(id))); - let result = restore_and_render( + let prepared = prepare( session, renderer, presentation, - RunOrigin::Public, - update.clone(), - { - clone!(session, renderer, update.table, workspace, load); - async move { - let OptionalUpdate::Update(name) = table else { - return Ok(()); - }; - - match &load { - Some(load) => { - rebind_table_task(&session, &renderer, &workspace, name, missing, load) - .await - }, - None => bind_table_task(&session, &workspace, name, missing).await, - } - } - }, + active, + plan, + overlay, + update, ) - .await; - - if let Some(load) = &load { - load.close(); - } + .await?; + let Outcome { committed, result } = + commit_and_render(session, renderer, presentation, origin, prepared).await; - if let Err(e) = &result { - match errors { - RestoreErrors::Publish => session.set_error(false, e.clone()).await?, - RestoreErrors::Suppress => { - let binding_after = ( - session.get_table().map(|t| t.get_name().to_owned()), - session.pending_table(), - ); - - if let Some(config) = rollback - && binding_after == binding_before - { - session.commit_view_config(config.into()).unwrap_or_log(); - } - }, - } + if let Err(e) = &result + && committed + && matches!(errors, RestoreErrors::Publish) + { + let _ = session.set_run_error(e.clone()).await; } result?; - if fresh { renderer.resize().await.unwrap_or_log(); } diff --git a/rust/perspective-viewer/src/rust/tasks/send_column_config.rs b/rust/perspective-viewer/src/rust/tasks/send_column_config.rs index f3d2f78613..378ea98e5b 100644 --- a/rust/perspective-viewer/src/rust/tasks/send_column_config.rs +++ b/rust/perspective-viewer/src/rust/tasks/send_column_config.rs @@ -15,7 +15,7 @@ use perspective_js::utils::*; use crate::config::ColumnConfigFieldUpdate; use crate::renderer::Renderer; -use crate::session::Session; +use crate::session::{EditDelta, OpKind, Session, StepOutcome}; /// Apply a [`ColumnConfigFieldUpdate`] from a column-style sidebar /// control to the active plugin's per-column config bucket on @@ -30,31 +30,50 @@ pub fn send_column_config( column_name: &str, update: ColumnConfigFieldUpdate, ) { - // Apply the renderer write synchronously so the StyleTab's - // revision-bump re-render path sees the new state immediately. - let view_config = session.get_view_config().clone(); - renderer.update_columns_config_field(&view_config, session, column_name.to_string(), update); - - clone!(session, renderer); - ApiFuture::spawn(async move { - let view_config_snapshot = session.get_view_config().clone(); - let columns_configs = renderer - .all_columns_configs_materialized(&view_config_snapshot, &session) - .await; - let plugin_token = - wasm_bindgen::JsValue::from_serde_ext(&renderer.get_plugin_config()).unwrap(); - renderer - .ensure_plugin_selected()? - .restore(&plugin_token, Some(&columns_configs))?; - - clone!(session); - renderer - .update_lazy(async move { Ok(session.get_view_with_dimensions()) }) - .await?; - renderer - .columns_config_changed - .emit(columns_configs.clone()); - renderer.column_style_changed.emit(columns_configs); - Ok(()) - }) + let column = column_name.to_string(); + let kind = OpKind::Edit { + delta: EditDelta::ColumnField { + column: column.clone(), + update: update.clone(), + }, + fields: None, + }; + + let ticket = session.submit(kind, { + clone!(session, renderer); + move |_ctx| { + Box::pin(async move { + let view_config = session.committed_view_config().clone(); + renderer.update_columns_config_field(&view_config, &session, column, update); + Ok(StepOutcome::Render(Box::pin(async move { + let view_config_snapshot = session.committed_view_config().clone(); + let columns_configs = renderer + .all_columns_configs_materialized(&view_config_snapshot, &session) + .await; + + let plugin_token = + wasm_bindgen::JsValue::from_serde_ext(&renderer.committed_plugin_config()) + .unwrap(); + + renderer + .ensure_plugin_selected()? + .restore(&plugin_token, Some(&columns_configs))?; + + clone!(session); + renderer + .update_lazy(async move { Ok(session.get_view_with_dimensions()) }) + .await?; + + renderer + .columns_config_changed + .emit(columns_configs.clone()); + + renderer.column_style_changed.emit(columns_configs); + Ok(()) + }))) + }) + } + }); + + ApiFuture::spawn(ticket.settle()); } diff --git a/rust/perspective-viewer/src/rust/tasks/send_plugin_config.rs b/rust/perspective-viewer/src/rust/tasks/send_plugin_config.rs index 08d0e4c626..b4f6b7f86c 100644 --- a/rust/perspective-viewer/src/rust/tasks/send_plugin_config.rs +++ b/rust/perspective-viewer/src/rust/tasks/send_plugin_config.rs @@ -15,7 +15,7 @@ use perspective_js::utils::*; use crate::config::ColumnConfigFieldUpdate; use crate::renderer::Renderer; -use crate::session::Session; +use crate::session::{EditDelta, OpKind, Session, StepOutcome}; /// Apply a [`ColumnConfigFieldUpdate`] from the Plugin-settings tab to /// the active plugin's bucket on [`Renderer`], then re-`restore` the @@ -28,27 +28,50 @@ use crate::session::Session; /// /// Column-style updates go through [`super::send_column_config`]. pub fn send_plugin_config(session: &Session, renderer: &Renderer, update: ColumnConfigFieldUpdate) { - let view_config = session.get_view_config().clone(); - let changed = renderer.update_plugin_config_field(&view_config, update); - clone!(session, renderer); - ApiFuture::spawn(async move { - if changed { - let plugin_config = renderer.get_plugin_config(); - let plugin_token = wasm_bindgen::JsValue::from_serde_ext(&plugin_config).unwrap(); - let view_config_snapshot = session.get_view_config().clone(); - let columns_configs = renderer - .all_columns_configs_materialized(&view_config_snapshot, &session) - .await; - renderer - .ensure_plugin_selected()? - .restore(&plugin_token, Some(&columns_configs))?; - clone!(session); - renderer - .update_lazy(async move { Ok(session.get_view_with_dimensions()) }) - .await?; - renderer.plugin_config_changed.emit(plugin_config); + let kind = OpKind::Edit { + delta: EditDelta::PluginField(update.clone()), + fields: None, + }; + + let ticket = session.submit(kind, { + clone!(session, renderer); + move |_ctx| { + Box::pin(async move { + let view_config = session.committed_view_config().clone(); + let changed = renderer.update_plugin_config_field(&view_config, update); + Ok(StepOutcome::Render(Box::pin(async move { + if changed { + deliver_plugin_config(&session, &renderer).await?; + } + + Ok(()) + }))) + }) } + }); + + ApiFuture::spawn(ticket.settle()); +} + +/// Re-`restore` the active plugin with its committed buckets, repaint, and +/// announce the plugin-level change. +pub(super) async fn deliver_plugin_config(session: &Session, renderer: &Renderer) -> ApiResult<()> { + let plugin_config = renderer.committed_plugin_config(); + let plugin_token = wasm_bindgen::JsValue::from_serde_ext(&plugin_config).unwrap(); + let view_config_snapshot = session.committed_view_config().clone(); + let columns_configs = renderer + .all_columns_configs_materialized(&view_config_snapshot, session) + .await; + + renderer + .ensure_plugin_selected()? + .restore(&plugin_token, Some(&columns_configs))?; + + clone!(session); + renderer + .update_lazy(async move { Ok(session.get_view_with_dimensions()) }) + .await?; - Ok(()) - }) + renderer.plugin_config_changed.emit(plugin_config); + Ok(()) } diff --git a/rust/perspective-viewer/src/rust/tasks/set_edit_mode.rs b/rust/perspective-viewer/src/rust/tasks/set_edit_mode.rs index 88efca75de..24fec8c10d 100644 --- a/rust/perspective-viewer/src/rust/tasks/set_edit_mode.rs +++ b/rust/perspective-viewer/src/rust/tasks/set_edit_mode.rs @@ -15,7 +15,7 @@ use perspective_js::utils::*; use crate::config::*; use crate::renderer::Renderer; -use crate::session::Session; +use crate::session::{EditDelta, OpKind, Session, StepOutcome}; /// Set the active plugin's `edit_mode`, persisting it in the [`Renderer`]'s /// plugin bucket and re-`restore`+rendering (the same merged-token path as @@ -27,35 +27,37 @@ use crate::session::Session; /// (e.g. charts) schema-gate the key out in [`Renderer::update_plugin_config`], /// making this a no-op for them. pub fn set_edit_mode(session: &Session, renderer: &Renderer, mode: &str) { - let view_config = session.get_view_config().clone(); let mut map = serde_json::Map::new(); map.insert( "edit_mode".to_owned(), serde_json::Value::String(mode.to_owned()), ); - let changed = renderer - .update_plugin_config(&view_config, OptionalUpdate::Update(map)) - .unwrap_or_default(); - clone!(session, renderer); - ApiFuture::spawn(async move { - if changed { - let plugin_config = renderer.get_plugin_config(); - let plugin_token = wasm_bindgen::JsValue::from_serde_ext(&plugin_config).unwrap(); - let view_config_snapshot = session.get_view_config().clone(); - let columns_configs = renderer - .all_columns_configs_materialized(&view_config_snapshot, &session) - .await; - renderer - .ensure_plugin_selected()? - .restore(&plugin_token, Some(&columns_configs))?; - clone!(session); - renderer - .update_lazy(async move { Ok(session.get_view_with_dimensions()) }) - .await?; - renderer.plugin_config_changed.emit(plugin_config); + let kind = OpKind::Edit { + delta: EditDelta::PluginConfig(map.clone()), + fields: None, + }; + + let ticket = session.submit(kind, { + clone!(session, renderer); + move |_ctx| { + Box::pin(async move { + let view_config = session.committed_view_config().clone(); + let changed = renderer + .update_plugin_config(&view_config, OptionalUpdate::Update(map)) + .unwrap_or_default(); + + Ok(StepOutcome::Render(Box::pin(async move { + if changed { + super::send_plugin_config::deliver_plugin_config(&session, &renderer) + .await?; + } + + Ok(()) + }))) + }) } + }); - Ok(()) - }) + ApiFuture::spawn(ticket.settle()); } diff --git a/rust/perspective-viewer/src/rust/tasks/table_lifecycle.rs b/rust/perspective-viewer/src/rust/tasks/table_lifecycle.rs index 3399fc9913..eddd6a5f85 100644 --- a/rust/perspective-viewer/src/rust/tasks/table_lifecycle.rs +++ b/rust/perspective-viewer/src/rust/tasks/table_lifecycle.rs @@ -19,8 +19,8 @@ use perspective_js::utils::{ApiFuture, ApiResult, LocalPollLoop}; use wasm_bindgen::JsValue; use super::pipeline::RunOrigin; -use super::restore_and_render::restore_and_render; -use crate::config::ViewerConfigUpdate; +use super::restore_panel::{Rebind, RestoreErrors, RestoreMode, restore_panel_step}; +use crate::config::{OptionalUpdate, ViewerConfigUpdate}; use crate::presentation::Presentation; use crate::utils::{AddListener, Subscription}; use crate::workspace::{Panel, Workspace}; @@ -113,52 +113,83 @@ pub(crate) async fn sweep_table_bindings( /// sweep's observation suspends its CURRENT state or no-ops. async fn suspend_panel(panel: &Panel) -> ApiResult<()> { clone!(panel.session, panel.renderer); - renderer - .clone() - .render_task(|_guard| async move { - if let Some(reset) = session.suspend_table() { - reset.await?; - if let Some(plugin) = renderer.active_plugin() { - plugin.clear().await?; - } - } + let ticket = panel.session.submit( + crate::session::OpKind::Restore { fields: None }, + move |_ctx| { + Box::pin(async move { + renderer + .clone() + .render_task(|_guard| async move { + if let Some(reset) = session.suspend_table() { + reset.await?; + if let Some(plugin) = renderer.active_plugin() { + plugin.clear().await?; + } + } + + Ok(()) + }) + .await?; - Ok(()) - }) - .await + Ok(crate::session::StepOutcome::Done) + }) + }, + ); + + ticket.settle().await } -/// Complete a PENDING panel's bind through the shared restore pipeline: the -/// committed view config is untouched, the pending name re-derived inside the -/// locked run (capture-free), the client federated exactly as a `restore` -/// would. A name still un-hosted (the sweep raced a delete) simply re-pends. +/// Complete a PENDING panel's bind, as the restore it is: the pending name +/// re-derived inside the op (capture-free), the client federated exactly as a +/// `restore` would, and the config and buckets committed as INTENT while the +/// table was away validated against the table that arrived. async fn bind_pending( panel: &Panel, workspace: &Workspace, presentation: &Presentation, ) -> ApiResult<()> { - restore_and_render( - &panel.session, - &panel.renderer, - presentation, - RunOrigin::Internal, - ViewerConfigUpdate::default(), - { - clone!(panel.session, workspace); - async move { + clone!(panel.session, panel.renderer, workspace, presentation); + let ticket = panel.session.submit( + crate::session::OpKind::Restore { fields: None }, + move |ctx| { + Box::pin(async move { let Some(name) = session.pending_table() else { - return Ok(()); + return Ok(crate::session::StepOutcome::Done); }; - super::restore_panel::bind_table_task( + let update = ViewerConfigUpdate { + table: OptionalUpdate::Update(name), + plugin_config: OptionalUpdate::Update(renderer.committed_plugin_config()), + columns_config: OptionalUpdate::Update(renderer.committed_columns_configs()), + ..ViewerConfigUpdate::default() + }; + + let result = restore_panel_step( + &ctx, &session, + &renderer, + &presentation, &workspace, - name, + RestoreMode::Existing { active: false }, + Rebind::Complete, + RunOrigin::Internal, + update, + RestoreErrors::Publish, crate::session::MissingTable::Pend, ) - .await - } + .await; + + if let Err(e) = &result + && session.pending_table().is_some() + { + let _ = session.set_run_error(e.clone()).await; + } + + result?; + Ok(crate::session::StepOutcome::Done) + }) }, - ) - .await + ); + + ticket.settle().await } diff --git a/rust/perspective-viewer/src/rust/tasks/transactional_restore.rs b/rust/perspective-viewer/src/rust/tasks/transactional_restore.rs new file mode 100644 index 0000000000..f4c71ed40d --- /dev/null +++ b/rust/perspective-viewer/src/rust/tasks/transactional_restore.rs @@ -0,0 +1,402 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +//! A restore as a TRANSACTION: [`prepare`] makes every check that can fail — +//! and waits on every round trip — without writing anything; [`commit`] then +//! replaces the panel's state in ONE swap and cannot fail. + +use std::rc::Rc; + +use perspective_client::config::ViewConfigUpdate; +use perspective_js::utils::*; + +use super::pipeline::{Committed, RunCommit, RunOrigin, RunSpec, locked_run, run_locked}; +use crate::config::{OptionalUpdate, SettingsUpdate, ThemeUpdate, ViewerConfigUpdate}; +use crate::presentation::Presentation; +use crate::renderer::{ + Renderer, ValidatedColumnsConfig, ValidatedPluginConfig, apply_columns_config_to, + apply_plugin_config_to, +}; +use crate::session::{ + BindPlan, BindingEffects, OverlayClause, PluginRef, PreparedView, Session, ViewDefaults, +}; + +/// A restore that has passed every check and only awaits [`commit`]. +pub(crate) struct Prepared { + state: PreparedState, + effects: Effects, +} + +/// The panel state a [`Prepared`] restore commits. +struct PreparedState { + view: PreparedView, + + /// The plugin selection to commit, when the restore ends on a plugin other + /// than the one selected (or selects the first). + plugin: Option, + + /// The name of the plugin the restore ends on — whose bucket the validated + /// bucket updates belong to. + target_name: String, + plugin_config: ValidatedPluginConfig, + columns_config: ValidatedColumnsConfig, + theme: Option>, + title: Option>, +} + +/// What a restore does to the ELEMENT rather than the panel — applied once the +/// restore can no longer be rejected, as part of rendering it. +pub(crate) struct Effects { + settings: SettingsUpdate, + host_theme: ThemeUpdate, +} + +/// Every fallible and asynchronous part of a restore. +pub(crate) async fn prepare( + session: &Session, + renderer: &Renderer, + presentation: &Presentation, + active: bool, + plan: BindPlan, + overlay: Option>>, + update: ViewerConfigUpdate, +) -> ApiResult { + let ViewerConfigUpdate { + plugin, + plugin_config, + columns_config, + settings, + theme, + title, + view_config, + .. + } = update; + + renderer.check_plugin_update(&plugin)?; + let named = renderer.resolve_plugin_update(&plugin); + let (target, swap_idx) = renderer.resolve_target(&plugin)?; + let defaults = match &named { + Some((_, metadata)) => ViewDefaults::Swap(metadata), + None => ViewDefaults::Rollup(&target.static_config), + }; + + let rebinds = !matches!(plan, BindPlan::Keep); + let view = session + .prepare_view(plan, view_config, defaults, overlay) + .await?; + let (plugin_config, columns_config) = renderer.prepare_buckets( + &target, + session, + view.config(), + view.view_schema(), + rebinds, + plugin_config, + columns_config, + )?; + + let panel_theme = match &theme { + OptionalUpdate::Update(theme) => Some(Some(theme.clone())), + OptionalUpdate::SetDefault => Some(presentation.get_default_theme_name().await), + OptionalUpdate::Missing if renderer.committed_theme().is_none() => { + Some(presentation.get_default_theme_name().await).filter(|x| x.is_some()) + }, + OptionalUpdate::Missing => None, + }; + + let title = match title { + OptionalUpdate::Update(title) => Some(Some(title).filter(|x| !x.is_empty())), + OptionalUpdate::SetDefault => Some(None), + OptionalUpdate::Missing => None, + }; + + Ok(Prepared { + state: PreparedState { + view, + plugin: swap_idx.map(|idx| renderer.plugin_ref(idx)).transpose()?, + target_name: target.static_config.name.clone(), + plugin_config, + columns_config, + theme: panel_theme, + title, + }, + effects: Effects { + settings, + host_theme: if active { + theme + } else { + OptionalUpdate::Missing + }, + }, + }) +} + +/// A UI edit as the transaction it is: the view-config `update` — and the +/// plugin swap `plugin_idx`, when the control made one — validated against what +/// the ops AHEAD of it left committed, including the server's `describe`. +pub(crate) async fn prepare_edit( + session: &Session, + renderer: &Renderer, + update: ViewConfigUpdate, + plugin_idx: Option, +) -> ApiResult { + let selected = renderer.committed_plugin_idx(); + let (target, swap_idx) = match plugin_idx.filter(|idx| Some(*idx) != selected) { + Some(idx) => (renderer.target_at(idx)?, Some(idx)), + None => renderer.resolve_target(&OptionalUpdate::Missing)?, + }; + + let view = session + .prepare_view(BindPlan::Keep, update, ViewDefaults::AsGiven, None) + .await? + .projected(); + + edit_of(renderer, view, &target.static_config.name, swap_idx) +} + +/// A new global-filter overlay as the transaction it is: the committed config +/// re-described with the clauses this panel's table honors. +pub(crate) async fn prepare_overlay( + session: &Session, + renderer: &Renderer, + overlay: Rc>, +) -> ApiResult { + let (target, swap_idx) = renderer.resolve_target(&OptionalUpdate::Missing)?; + let view = session + .prepare_view( + BindPlan::Keep, + ViewConfigUpdate::default(), + ViewDefaults::AsGiven, + Some(overlay), + ) + .await?; + + edit_of(renderer, view, &target.static_config.name, swap_idx) +} + +fn edit_of( + renderer: &Renderer, + view: PreparedView, + target_name: &str, + swap_idx: Option, +) -> ApiResult { + Ok(Prepared { + state: PreparedState { + view, + plugin: swap_idx.map(|idx| renderer.plugin_ref(idx)).transpose()?, + target_name: target_name.to_owned(), + plugin_config: ValidatedPluginConfig::Missing, + columns_config: ValidatedColumnsConfig::Missing, + theme: None, + title: None, + }, + effects: Effects { + settings: OptionalUpdate::Missing, + host_theme: OptionalUpdate::Missing, + }, + }) +} + +/// Commit a [`prepare_edit`] NOW, in the op's step — so the write is the +/// drain's, and the render that follows may be detached. +pub(crate) fn commit_edit(session: &Session, renderer: &Renderer, prepared: Prepared) -> Committed { + let (committed, binding) = commit(session, renderer, prepared.state); + debug_assert!(!binding.rebound()); + binding.forget(); + committed +} + +/// Commit and render a [`Prepared`] restore under a draw lock the caller +/// ALREADY holds (`load()`, which holds it across its payload's arrival). +pub(crate) async fn commit_and_render_locked( + guard: crate::utils::RenderGuard, + session: &Session, + renderer: &Renderer, + Prepared { state, .. }: Prepared, +) -> ApiResult<()> { + let commit = { + let (session, renderer) = (session.clone(), renderer.clone()); + Box::new(move || commit(&session, &renderer, state)) + }; + + run_locked(guard, session.clone(), renderer.clone(), RunSpec { + origin: RunOrigin::Public, + commit: RunCommit::Deferred(commit), + presentation: None, + }) + .await +} + +/// Replace the panel's state with a [`Prepared`] restore, in one swap. +fn commit( + session: &Session, + renderer: &Renderer, + prepared: PreparedState, +) -> (Committed, BindingEffects) { + let PreparedState { + view, + plugin, + target_name, + plugin_config, + columns_config, + theme, + title, + } = prepared; + + let had_plugin = renderer.active_plugin().is_some(); + let activate = plugin.is_some(); + let mut plugin_config_changed = false; + let mut columns_config_changed = false; + let binding = session.commit_view(view, |mut next| { + if let Some(plugin) = plugin { + next = next.with_plugin(plugin); + } + + let mut bucket = next.bucket(&target_name); + plugin_config_changed = apply_plugin_config_to(&mut bucket, plugin_config); + columns_config_changed = apply_columns_config_to(&mut bucket, columns_config); + if plugin_config_changed || columns_config_changed { + next = next.with_bucket(&target_name, bucket); + } + + if let Some(theme) = theme { + next = next.with_theme(theme); + } + + if let Some(title) = title { + next = next.with_title(title); + } + + next + }); + + let committed = Committed { + activate, + plugin_swapped: activate && had_plugin, + plugin_config_changed, + columns_config_changed, + }; + + (committed, binding) +} + +/// A restore that leaves the table binding alone, for the host's own re-renders +/// (a reset, an auto-pause resume): to be called only from a running op's step. +pub(crate) async fn restore_in_place( + session: &Session, + renderer: &Renderer, + presentation: &Presentation, + origin: RunOrigin, + update: ViewerConfigUpdate, +) -> ApiResult<()> { + let prepared = prepare( + session, + renderer, + presentation, + false, + BindPlan::Keep, + None, + update, + ) + .await?; + commit_and_render(session, renderer, presentation, origin, prepared) + .await + .result +} + +/// Whether a restore that failed had ALREADY committed — what tells a rejection +/// (the panel is as it was) from a failure to render valid state. +pub(crate) struct Outcome { + pub committed: bool, + pub result: ApiResult<()>, +} + +/// Commit a [`Prepared`] restore and render it: element-level effects, then the +/// locked run, whose first act is the commit — under the draw lock, so no +/// render of the outgoing state can bind a `View` over the incoming table. +pub(crate) async fn commit_and_render( + session: &Session, + renderer: &Renderer, + presentation: &Presentation, + origin: RunOrigin, + Prepared { state, effects }: Prepared, +) -> Outcome { + let Effects { + settings, + host_theme, + } = effects; + + let did_commit = std::rc::Rc::new(std::cell::Cell::new(false)); + let result = render( + session, + renderer, + presentation, + origin, + settings, + host_theme, + { + let (session, renderer, did_commit) = + (session.clone(), renderer.clone(), did_commit.clone()); + Box::new(move || { + did_commit.set(true); + commit(&session, &renderer, state) + }) + }, + ) + .await; + + Outcome { + committed: did_commit.get(), + result, + } +} + +#[allow(clippy::too_many_arguments)] +async fn render( + session: &Session, + renderer: &Renderer, + presentation: &Presentation, + origin: RunOrigin, + settings: SettingsUpdate, + host_theme: ThemeUpdate, + commit: Box (Committed, BindingEffects)>, +) -> ApiResult<()> { + if let OptionalUpdate::Update(x) = settings { + presentation.set_settings_attribute(x); + presentation.set_settings_before_open(x); + } + + match host_theme { + OptionalUpdate::SetDefault => { + if presentation.get_selected_theme_name().await.is_some() { + presentation.set_theme_name(None).await?; + } + }, + OptionalUpdate::Update(x) => { + presentation.set_theme_name(Some(&x)).await?; + }, + OptionalUpdate::Missing => {}, + } + + locked_run(session, renderer, RunSpec { + origin, + commit: RunCommit::Deferred(commit), + presentation: Some(presentation.clone()), + }) + .await?; + + if renderer.needs_restyle() { + renderer.restyle_all().await?; + } + + presentation.publish_theme_config().await?; + Ok(()) +} diff --git a/rust/perspective-viewer/src/rust/workspace.rs b/rust/perspective-viewer/src/rust/workspace.rs index a5b8f47527..7467e37927 100644 --- a/rust/perspective-viewer/src/rust/workspace.rs +++ b/rust/perspective-viewer/src/rust/workspace.rs @@ -19,7 +19,7 @@ use perspective_client::Client; use perspective_client::config::Filter; use crate::renderer::Renderer; -use crate::session::Session; +use crate::session::{OverlayClause, Session}; use crate::utils::{EffectLedger, PubSub, Subscription, spawn_owned}; /// A unique identifier for a [`Panel`] within a [`Workspace`]. @@ -54,22 +54,37 @@ impl From<&str> for PanelId { #[derive(Default)] struct GlobalFilterSet { restored: Vec, - contributions: Vec<(PanelId, Vec)>, + + /// Each master's selection, every clause typed as the MASTER types its + /// column — what decides which listeners it applies to. + contributions: Vec<(PanelId, Vec)>, } impl GlobalFilterSet { /// The effective set: `restored`, then each contribution in order, /// deduped by clause equality. fn flatten(&self) -> Vec { - let mut flat: Vec = Vec::new(); + self.overlay().into_iter().map(|x| x.filter).collect() + } + + /// [`Self::flatten`] with each clause's broadcaster type — untyped for the + /// restored bucket, which has no broadcaster. + fn overlay(&self) -> Vec { + let mut flat: Vec = Vec::new(); let all = self .restored .iter() - .chain(self.contributions.iter().flat_map(|(_, fs)| fs.iter())); + .cloned() + .map(OverlayClause::from) + .chain( + self.contributions + .iter() + .flat_map(|(_, fs)| fs.iter().cloned()), + ); - for filter in all { - if !flat.contains(filter) { - flat.push(filter.clone()); + for clause in all { + if !flat.iter().any(|x| x.filter == clause.filter) { + flat.push(clause); } } @@ -86,7 +101,8 @@ impl GlobalFilterSet { } /// Replace `id`'s contribution. - fn set_contribution(&mut self, id: &PanelId, filters: Vec) -> bool { + fn set_contribution(&mut self, id: &PanelId, filters: Vec>) -> bool { + let filters = filters.into_iter().map(Into::into).collect::>(); self.with_change(|s| { if filters.is_empty() { s.contributions.retain(|(pid, _)| pid != id); @@ -109,14 +125,14 @@ impl GlobalFilterSet { let owners = self .contributions .iter() - .filter(|(_, fs)| fs.contains(&clause)) + .filter(|(_, fs)| fs.iter().any(|f| f.filter == clause)) .map(|(pid, _)| pid.clone()) .collect(); let changed = self.with_change(|s| { s.restored.retain(|f| f != &clause); for (_, fs) in s.contributions.iter_mut() { - fs.retain(|f| f != &clause); + fs.retain(|f| f.filter != clause); } s.contributions.retain(|(_, fs)| !fs.is_empty()); @@ -477,6 +493,12 @@ impl Workspace { self.0.borrow().filters.flatten() } + /// [`Self::global_filters`] as broadcast to the panels: each clause with + /// the type its column has in the master that selected it. + pub fn overlay(&self) -> Vec { + self.0.borrow().filters.overlay() + } + /// A handle to the `filters_changed` PubSub (fires after any change to the /// global filter set). pub fn filters_changed(&self) -> Rc> { @@ -507,13 +529,13 @@ impl Workspace { /// Replace master `id`'s contribution with a new selection (empty = /// deselect). See [`GlobalFilterSet::set_contribution`]. - pub fn set_contribution(&self, id: &PanelId, filters: Vec) { + pub fn set_contribution(&self, id: &PanelId, filters: Vec) { self.mutate_filters(|s| (s.set_contribution(id, filters), ())); } /// Drop master `id`'s contribution (deselect / demote / close). pub fn clear_contribution(&self, id: &PanelId) { - self.mutate_filters(|s| (s.set_contribution(id, Vec::new()), ())); + self.mutate_filters(|s| (s.set_contribution(id, Vec::::new()), ())); } /// Remove the effective-set clause at `index`. @@ -805,7 +827,9 @@ impl Workspace { pub fn remove_panel(&self, id: &PanelId) -> Option { let (removed, changed, filters_pubsub, was_staged, staged_pubsub) = { let mut data = self.0.borrow_mut(); - let changed = data.filters.set_contribution(id, Vec::new()); + let changed = data + .filters + .set_contribution(id, Vec::::new()); let (removed, was_staged) = match data.panels.iter().position(|p| &p.panel.id == id) { Some(idx) => { let entry = data.panels.remove(idx); @@ -1049,7 +1073,7 @@ mod tests { // Re-selecting the same value reports no visible change. assert!(!s.set_contribution(&p("x"), vec![f("a", "2")])); // Empty = deselect: the entry is removed. - assert!(s.set_contribution(&p("x"), Vec::new())); + assert!(s.set_contribution(&p("x"), Vec::::new())); assert_eq!(s.flatten(), Vec::::new()); } @@ -1059,7 +1083,7 @@ mod tests { s.set_contribution(&p("x"), vec![f("a", "1")]); s.set_contribution(&p("y"), vec![f("b", "2")]); // Clearing one master's contribution leaves the other's intact. - assert!(s.set_contribution(&p("x"), Vec::new())); + assert!(s.set_contribution(&p("x"), Vec::::new())); assert_eq!(s.flatten(), vec![f("b", "2")]); } @@ -1083,7 +1107,7 @@ mod tests { let mut s = GlobalFilterSet::default(); s.set_restored(vec![f("a", "1")]); // A deselect (empty contribution) does NOT drop the restored bucket. - assert!(!s.set_contribution(&p("x"), Vec::new())); + assert!(!s.set_contribution(&p("x"), Vec::::new())); assert_eq!(s.flatten(), vec![f("a", "1")]); // A real selection replaces it. assert!(s.set_contribution(&p("x"), vec![f("b", "2")])); diff --git a/rust/perspective-viewer/src/ts/plugin.ts b/rust/perspective-viewer/src/ts/plugin.ts index f8ac1ee7a0..71f00687f2 100644 --- a/rust/perspective-viewer/src/ts/plugin.ts +++ b/rust/perspective-viewer/src/ts/plugin.ts @@ -97,6 +97,9 @@ export interface IPerspectiveViewerPlugin { * The per-column Style-tab schema: the controls the viewer renders * for one column, re-queried on every field edit so the field set * may depend on `current_value`. + * + * Must be a function of its arguments alone, never of what this element + * last drew or was last `restore`d with. */ column_config_schema?( view_type: string, @@ -104,7 +107,7 @@ export interface IPerspectiveViewerPlugin { column_name: string, current_value: Record | null, viewer_config?: Record, - column_stats?: Record, + plugin_config?: Record | null, ): { fields: Array & { kind: string }> }; /** diff --git a/rust/perspective-viewer/test/js/dragdrop/drop_column_settings.spec.ts b/rust/perspective-viewer/test/js/dragdrop/drop_column_settings.spec.ts index 483b13bda8..ddfc84296f 100644 --- a/rust/perspective-viewer/test/js/dragdrop/drop_column_settings.spec.ts +++ b/rust/perspective-viewer/test/js/dragdrop/drop_column_settings.spec.ts @@ -58,6 +58,10 @@ test.describe("Drag and Drop", () => { await configUpdated(); const config = await view.save(); expect(config.columns).toEqual(["Category", "Sales"]); + await expect(view.container.locator("#add-expression")).toHaveClass( + /dragdrop-hover/, + ); + const contents = await getSettingsPanelContents(page); await compareContentsToSnapshot(contents); }); @@ -124,6 +128,10 @@ test.describe("Drag and Drop", () => { const config = await view.save(); expect(config.group_by).toEqual(["Category"]); expect(config.columns).toEqual(["Sales", "Profit"]); + await expect(view.container.locator("#add-expression")).toHaveClass( + /dragdrop-hover/, + ); + const contents = await getSettingsPanelContents(page); await compareContentsToSnapshot(contents); }); @@ -150,6 +158,10 @@ test.describe("Drag and Drop", () => { const config = await view.save(); expect(config.split_by).toEqual(["Category"]); expect(config.columns).toEqual(["Sales"]); + await expect(view.container.locator("#add-expression")).toHaveClass( + /dragdrop-hover/, + ); + const contents = await getSettingsPanelContents(page); await compareContentsToSnapshot(contents); }); @@ -176,6 +188,10 @@ test.describe("Drag and Drop", () => { const config = await view.save(); expect(config.filter).toEqual([["Category", "==", null]]); expect(config.columns).toEqual(["Sales"]); + await expect(view.container.locator("#add-expression")).toHaveClass( + /dragdrop-hover/, + ); + const contents = await getSettingsPanelContents(page); await compareContentsToSnapshot(contents); }); @@ -202,6 +218,10 @@ test.describe("Drag and Drop", () => { const config = await view.save(); expect(config.sort).toEqual([["Category", "asc"]]); expect(config.columns).toEqual(["Sales"]); + await expect(view.container.locator("#add-expression")).toHaveClass( + /dragdrop-hover/, + ); + const contents = await getSettingsPanelContents(page); await compareContentsToSnapshot(contents); }); diff --git a/rust/perspective-viewer/test/js/multi_panel/global_filter.spec.ts b/rust/perspective-viewer/test/js/multi_panel/global_filter.spec.ts index e56591ac14..2d3a3598a8 100644 --- a/rust/perspective-viewer/test/js/multi_panel/global_filter.spec.ts +++ b/rust/perspective-viewer/test/js/multi_panel/global_filter.spec.ts @@ -601,3 +601,134 @@ test.describe("Global filters: persistence", () => { expect((await save(page)).masters ?? []).toEqual([]); }); }); + +const FOREIGN_TABLES = { + "a table without the column": "City,Units\nAustin,1\nDallas,2\nReno,3", + "a table typing the column differently": "State,Units\n1,10\n2,20\n3,30", +}; + +async function restore_with_foreign(page, csv) { + await page.evaluate(async (csv) => { + const worker = (window as any).__TEST_WORKER__; + const names = await worker.get_hosted_table_names(); + if (!names.includes("gf-foreign")) { + await worker.table(csv, { name: "gf-foreign" }); + } + }, csv); + + await restore(page, { + ...TRI_CONFIG, + panels: { + one: { table: TABLE, title: "One" }, + two: { table: TABLE, title: "Two" }, + three: { table: "gf-foreign", title: "Three" }, + }, + masters: ["one"], + }); +} + +test.describe("Global filters: listeners that cannot honor a clause", () => { + for (const [name, csv] of Object.entries(FOREIGN_TABLES)) { + test(`a listener on ${name} ignores the clause and keeps rendering`, async ({ + page, + }) => { + await restore_with_foreign(page, csv); + const master = await id_by_title(page, "One"); + const detail = await id_by_title(page, "Two"); + const foreign = await id_by_title(page, "Three"); + const baseline = await num_rows(page, detail); + const foreign_rows = await num_rows(page, foreign); + await dispatch_select(page, { + panel: master, + selected: true, + insertFilters: [["State", "==", "Texas"]], + }); + + await wait_rows_below(page, detail, baseline); + await page.evaluate(async () => { + await ( + document.querySelector("perspective-viewer") as any + ).flush(); + }); + + expect(await num_rows(page, foreign)).toBe(foreign_rows); + expect(foreign_rows).toBe(3); + expect((await save_panel(page, foreign)).filter).toEqual([]); + expect((await save(page)).global_filters).toEqual([ + ["State", "==", "Texas"], + ]); + + await expect( + page.locator("perspective-viewer span#status.errored"), + ).toHaveCount(0); + }); + } + + test("a listener rebound to a table that has the column starts filtering with no new selection", async ({ + page, + }) => { + await restore_with_foreign( + page, + FOREIGN_TABLES["a table without the column"], + ); + + const master = await id_by_title(page, "One"); + const detail = await id_by_title(page, "Two"); + const foreign = await id_by_title(page, "Three"); + const baseline = await num_rows(page, detail); + await dispatch_select(page, { + panel: master, + selected: true, + insertFilters: [["State", "==", "Texas"]], + }); + + await wait_rows_below(page, detail, baseline); + const filtered = await num_rows(page, detail); + await page.evaluate( + async ({ id, table }) => { + const viewer = document.querySelector( + "perspective-viewer", + ) as any; + await viewer.restore({ table }, { panel: id }); + }, + { id: foreign, table: TABLE }, + ); + + expect(await num_rows(page, foreign)).toBe(filtered); + expect((await save_panel(page, foreign)).filter).toEqual([]); + }); + + test("a listener rebound to a table that lacks the column stops filtering", async ({ + page, + }) => { + await restore_with_foreign( + page, + FOREIGN_TABLES["a table without the column"], + ); + + const master = await id_by_title(page, "One"); + const detail = await id_by_title(page, "Two"); + const baseline = await num_rows(page, detail); + await dispatch_select(page, { + panel: master, + selected: true, + insertFilters: [["State", "==", "Texas"]], + }); + + await wait_rows_below(page, detail, baseline); + await page.evaluate( + async ({ id }) => { + const viewer = document.querySelector( + "perspective-viewer", + ) as any; + await viewer.restore({ table: "gf-foreign" }, { panel: id }); + }, + { id: detail }, + ); + + expect(await num_rows(page, detail)).toBe(3); + await expect( + page.locator("perspective-viewer span#status.errored"), + ).toHaveCount(0); + }); +}); diff --git a/rust/perspective-viewer/test/js/multi_panel/load_restore_race.spec.ts b/rust/perspective-viewer/test/js/multi_panel/load_restore_race.spec.ts index 6fcde244d7..e60121c242 100644 --- a/rust/perspective-viewer/test/js/multi_panel/load_restore_race.spec.ts +++ b/rust/perspective-viewer/test/js/multi_panel/load_restore_race.spec.ts @@ -30,11 +30,23 @@ test.beforeEach(async ({ page }) => { armInvariants(test); -for (const plugin of ["Datagrid", "Debug"]) { +const PLUGIN_PAGES = [ + ["Datagrid", "superstore-all.html"], + ["Debug", "superstore.html"], +]; + +for (const [plugin, html] of PLUGIN_PAGES) { test(`load + immediate restore holds \`columns\` across ${VIEWER_COUNT} viewers (${plugin})`, async ({ page, }) => { test.setTimeout(120_000); + await page.goto(`/rust/perspective-viewer/test/html/${html}`); + await page.evaluate(async () => { + while (!window["__TEST_PERSPECTIVE_READY__"]) { + await new Promise((x) => setTimeout(x, 10)); + } + }); + const results = await page.evaluate( async ({ tableName, columns, plugin, count }) => { const worker = (window as any).__TEST_WORKER__; @@ -123,7 +135,7 @@ test("public mutators are render-quiescent at resolution (I6)", async ({ await quiesce("load", () => v.load(table)); await quiesce("restore", () => - v.restore({ plugin: "Datagrid", columns: ["Sales"] }), + v.restore({ plugin: "Debug", columns: ["Sales"] }), ); await quiesce("reset", () => v.reset()); await quiesce("restore", () => diff --git a/rust/perspective-viewer/test/js/status_indicator/updating.spec.ts b/rust/perspective-viewer/test/js/status_indicator/updating.spec.ts index 689ef89cc1..4ea51dc389 100644 --- a/rust/perspective-viewer/test/js/status_indicator/updating.spec.ts +++ b/rust/perspective-viewer/test/js/status_indicator/updating.spec.ts @@ -121,7 +121,7 @@ test.describe("StatusIndicator 'updating' settles", () => { } }); - test("after an errored run is reset (T6)", async ({ page }) => { + test("after a rejected restore (T6)", async ({ page }) => { await goto(page, "/rust/perspective-viewer/test/html/superstore.html"); await open_settings(page); const status = page.locator("perspective-viewer #status_reconnect"); @@ -131,12 +131,10 @@ test.describe("StatusIndicator 'updating' settles", () => { await viewer.restore({ expressions: { broken: 'upper("Sales")' }, }); - } catch { - // The restore run fails; error state is the expectation. - } + } catch {} }); - await expect(status).toHaveClass(/errored/, { timeout: 10_000 }); + await expect(status).not.toHaveClass(/errored/); await assert_settled(page); }); diff --git a/rust/perspective-viewer/test/js/viewer_api/op_queue.spec.ts b/rust/perspective-viewer/test/js/viewer_api/op_queue.spec.ts new file mode 100644 index 0000000000..839dd81eb2 --- /dev/null +++ b/rust/perspective-viewer/test/js/viewer_api/op_queue.spec.ts @@ -0,0 +1,235 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +import { test, expect } from "../helpers.ts"; + +test.beforeEach(async ({ page }) => { + await page.goto("/rust/perspective-viewer/test/html/superstore.html"); + await page.evaluate(async () => { + while (!window["__TEST_PERSPECTIVE_READY__"]) { + await new Promise((x) => setTimeout(x, 10)); + } + }); + + await page.evaluate(async () => { + await document.querySelector("perspective-viewer")!.restore({ + plugin: "Debug", + }); + }); +}); + +test.describe("op queue", () => { + test("unawaited restores land in call order", async ({ page }) => { + const config = await page.evaluate(async () => { + const viewer = document.querySelector("perspective-viewer")! as any; + const first = viewer.restore({ group_by: ["State"] }); + const second = viewer.restore({ split_by: ["Category"] }); + const third = viewer.restore({ group_by: ["Region"] }); + await Promise.all([first, second, third]); + return await viewer.save(); + }); + + expect(config.group_by).toEqual(["Region"]); + expect(config.split_by).toEqual(["Category"]); + }); + + test("a restore covered by a later one resolves with it", async ({ + page, + }) => { + const result = await page.evaluate(async () => { + const viewer = document.querySelector("perspective-viewer")! as any; + const first = viewer.restore({ group_by: ["State"] }); + const second = viewer.restore({ + group_by: ["Region"], + sort: [["Sales", "desc"]], + }); + + const settled = await Promise.allSettled([first, second]); + const config = await viewer.save(); + return { + statuses: settled.map((x) => x.status), + group_by: config.group_by, + sort: config.sort, + }; + }); + + expect(result.statuses).toEqual(["fulfilled", "fulfilled"]); + expect(result.group_by).toEqual(["Region"]); + expect(result.sort).toEqual([["Sales", "desc"]]); + }); + + test("a covering restore that fails rejects the one it covered", async ({ + page, + }) => { + const result = await page.evaluate(async () => { + const viewer = document.querySelector("perspective-viewer")! as any; + const before = await viewer.save(); + const first = viewer.restore( + { group_by: ["State"] }, + { suppress_errors: true }, + ); + + const second = viewer.restore( + { group_by: ["Not A Column"] }, + { suppress_errors: true }, + ); + + const settled = await Promise.allSettled([first, second]); + const after = await viewer.save(); + return { + statuses: settled.map((x) => x.status), + before: before.group_by, + after: after.group_by, + }; + }); + + expect(result.statuses).toEqual(["rejected", "rejected"]); + expect(result.after).toEqual(result.before); + }); + + test("restores with partially overlapping fields both apply", async ({ + page, + }) => { + const config = await page.evaluate(async () => { + const viewer = document.querySelector("perspective-viewer")! as any; + const first = viewer.restore({ + group_by: ["State"], + sort: [["Sales", "desc"]], + }); + + const second = viewer.restore({ + group_by: ["Region"], + split_by: ["Category"], + }); + + await Promise.all([first, second]); + return await viewer.save(); + }); + + expect(config.group_by).toEqual(["Region"]); + expect(config.split_by).toEqual(["Category"]); + expect(config.sort).toEqual([["Sales", "desc"]]); + }); + + test("a restore made during a slow load applies after it", async ({ + page, + }) => { + const result = await page.evaluate(async () => { + const viewer = document.querySelector("perspective-viewer")! as any; + const table = await viewer.getTable(); + let release: (x: unknown) => void = () => {}; + const slow = new Promise((resolve) => { + release = resolve; + }); + + const order: string[] = []; + const load = viewer.load(slow).then(() => order.push("load")); + const restore = viewer + .restore({ group_by: ["State"] }) + .then(() => order.push("restore")); + + await new Promise((x) => setTimeout(x, 50)); + const pending = order.length; + release(table); + await Promise.all([load, restore]); + const config = await viewer.save(); + return { pending, order, group_by: config.group_by }; + }); + + expect(result.pending).toBe(0); + expect(result.order).toEqual(["load", "restore"]); + expect(result.group_by).toEqual(["State"]); + }); + + test("a later load supersedes an earlier one that has not resolved", async ({ + page, + }) => { + const result = await page.evaluate(async () => { + const viewer = document.querySelector("perspective-viewer")! as any; + const table = await viewer.getTable(); + let release: (x: unknown) => void = () => {}; + const slow = new Promise((resolve) => { + release = resolve; + }); + + const first = viewer.load(slow); + const second = viewer.load(table); + release(table); + const settled = await Promise.allSettled([first, second]); + const loaded = await viewer.getTable(); + return { + statuses: settled.map((x) => x.status), + same: (await loaded.get_name()) === (await table.get_name()), + }; + }); + + expect(result.statuses).toEqual(["fulfilled", "fulfilled"]); + expect(result.same).toBe(true); + }); + + test("every commit a restore or load makes is described before it renders", async ({ + page, + }) => { + const warnings: string[] = []; + page.on("console", (msg) => { + if (msg.text().includes("was not described")) { + warnings.push(msg.text()); + } + }); + + await page.evaluate(async () => { + const viewer = document.querySelector("perspective-viewer")! as any; + const table = await viewer.getTable(); + await viewer.restore({ + group_by: ["State"], + columns: ["Sales", "x"], + expressions: { x: '"Sales" * 2' }, + }); + + await viewer.restore({ plugin: "Debug", split_by: ["Category"] }); + await viewer.load(table); + await viewer.restore({ group_by: ["Region"] }); + await viewer.flush(); + }); + + expect(warnings).toEqual([]); + }); + + test("the busy indicator is on while an op is queued and off once it settles", async ({ + page, + }) => { + const indicator = page.locator( + "perspective-viewer span#status_updating", + ); + await page.evaluate(async () => { + const viewer = document.querySelector("perspective-viewer")! as any; + await viewer.restore({ settings: true }); + const table = await viewer.getTable(); + const slow = new Promise((resolve) => { + (window as any).__RELEASE__ = () => resolve(table); + }); + + (window as any).__LOAD__ = viewer.load(slow); + }); + + await expect(indicator).toHaveClass(/updating/); + await page.evaluate(async () => { + (window as any).__RELEASE__(); + await (window as any).__LOAD__; + await ( + document.querySelector("perspective-viewer")! as any + ).flush(); + }); + + await expect(indicator).not.toHaveClass(/updating/); + }); +}); diff --git a/rust/perspective-viewer/test/js/viewer_api/table_lifecycle.spec.ts b/rust/perspective-viewer/test/js/viewer_api/table_lifecycle.spec.ts index 6e8a57ee6c..3b5b1d6448 100644 --- a/rust/perspective-viewer/test/js/viewer_api/table_lifecycle.spec.ts +++ b/rust/perspective-viewer/test/js/viewer_api/table_lifecycle.spec.ts @@ -289,8 +289,8 @@ test.describe("Reactive table lifecycle", () => { expect(result.table).toBe("load-viewer-csv"); expect(result.bound).toBe("load-viewer-csv"); await expect( - page.locator("perspective-viewer span#status"), - ).toHaveClass(/errored/); + page.locator("perspective-viewer span#status.errored"), + ).toHaveCount(0); }); test("addPanel({table}) with an un-hosted name rejects by default", async ({ @@ -310,6 +310,131 @@ test.describe("Reactive table lifecycle", () => { }); expect(result.addError).toContain('Unknown table "lifecycle-t7"'); - expect(result.after).toBe(result.before + 1); + expect(result.after).toBe(result.before); + }); + + test("addPanel() with a config its table rejects creates no panel", async ({ + page, + }) => { + const result = await page.evaluate(async () => { + const viewer = document.querySelector("perspective-viewer") as any; + const before = viewer.getPanelNames().length; + let addError: string | null = null; + try { + await viewer.addPanel({ + table: "load-viewer-csv", + group_by: ["Not A Column"], + }); + } catch (e) { + addError = String(e); + } + + return { addError, before, after: viewer.getPanelNames().length }; + }); + + expect(result.addError).toContain("Not A Column"); + expect(result.after).toBe(result.before); + }); + + test("restore({table}) with a config the incoming table rejects leaves the panel bound to the outgoing one", async ({ + page, + }) => { + const result = await page.evaluate(async () => { + const worker = (window as any).__TEST_WORKER__; + const viewer = document.querySelector("perspective-viewer") as any; + await worker.table("a,b\n1,x\n2,y", { name: "lifecycle-t8" }); + await viewer.restore({ group_by: ["State"], columns: ["Sales"] }); + const before = await viewer.save(); + const view = await viewer.getView(); + let restoreError: string | null = null; + try { + await viewer.restore({ + table: "lifecycle-t8", + columns: ["a", "Sales"], + }); + } catch (e) { + restoreError = String(e); + } + + const table = await viewer.getTable(); + return { + restoreError, + before, + after: await viewer.save(), + bound: await table.get_name(), + rows: await view.num_rows(), + }; + }); + + expect(result.restoreError).toContain("Sales"); + expect(result.bound).toBe("load-viewer-csv"); + expect(result.after).toEqual(result.before); + expect(result.rows).toBeGreaterThan(0); + await expect( + page.locator("perspective-viewer span#status.errored"), + ).toHaveCount(0); + }); + + test("restore({table}) with an expression the incoming table rejects leaves the panel bound to the outgoing one", async ({ + page, + }) => { + const result = await page.evaluate(async () => { + const worker = (window as any).__TEST_WORKER__; + const viewer = document.querySelector("perspective-viewer") as any; + await worker.table("a,b\n1,x\n2,y", { name: "lifecycle-t9" }); + const before = await viewer.save(); + let restoreError: string | null = null; + try { + await viewer.restore({ + table: "lifecycle-t9", + columns: ["a", "bad"], + expressions: { bad: '"Sales" + 1' }, + }); + } catch (e) { + restoreError = String(e); + } + + const table = await viewer.getTable(); + return { + restoreError, + before, + after: await viewer.save(), + bound: await table.get_name(), + }; + }); + + expect(result.restoreError).not.toBeNull(); + expect(result.bound).toBe("load-viewer-csv"); + expect(result.after).toEqual(result.before); + }); + + test("a pending restore whose config the arriving table rejects never binds, and shows the error", async ({ + page, + }) => { + const result = await page.evaluate(async () => { + const worker = (window as any).__TEST_WORKER__; + const viewer = document.querySelector("perspective-viewer") as any; + await viewer.restore( + { table: "lifecycle-t10", group_by: ["Not A Column"] }, + { wait_for_table: true }, + ); + + const pending = await viewer.save(); + await worker.table("a,b\n1,x\n2,y", { name: "lifecycle-t10" }); + await new Promise((x) => setTimeout(x, 1000)); + const table = await viewer.getTable().catch(() => null); + return { + pending: pending.group_by, + table: (await viewer.save()).table, + bound: table ? await table.get_name() : null, + }; + }); + + expect(result.pending).toEqual(["Not A Column"]); + expect(result.table).toBe("lifecycle-t10"); + expect(result.bound).toBeNull(); + await expect( + page.locator("perspective-viewer span#status"), + ).toHaveClass(/errored/); }); });