From 657b98afe63ec3b46631ee541d0d59a9e9646ece Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 00:30:02 +1100 Subject: [PATCH 01/21] feat(stack-encrypt,eql): store a JSON document through a data plan as EncryptedJson The JSON index could only be reached from typed Rust: a data plan parsed `{"json": ...}` and refused it (json_index_pending), and the json_tree family was missing from the EQL catalog, so no binding could name it. The catalog now lists the family once, as a document family (eql_domains::DOCUMENT_FAMILIES): its stored type is the target `EncryptedJson` (public.eql_v4_json_tree), produced from the new `json` value kind and carrying the `json` index, with EncryptedJsonQuery, EncryptedJsonLeafQuery and EncryptedJsonEntry beside it. eql-codegen renders the target table row, v4::all()/all_query() and the Go types from it; the scalar SQL and struct materializers never see it, since its SQL and Rust bodies are hand-written and v4 only. A plan field names the target with the JSON index's options, `"target": {"name": "EncryptedJson", "options": {...}}` (the bare name for the defaults). The plan resolves the options once, when it is built, so the field's writes, queries and reads run under one definition, and refuses options on a target that takes none, a json field with no target, and a sealed null on a document. TargetResolver::encrypt takes the options; eql-bindings' encrypt_with seals the document through the same JsonTree layout typed Rust uses. The document crosses the binding as UTF-8 JSON text and is parsed once, by json::parse_text: an object that names a member twice and an integer literal outside the 64-bit range are refused rather than resolved or rounded, and every refusal carries a code and a position, never the text (TargetError::Json). It opens back to its text in a bytes leaf. tests/json_tree_data_plan.rs holds the data plan to the typed layout: the same selectors, ordering terms and containment projection for the same document under the same column, each path opening what the other sealed, and a batch of documents in one key request. Refs #1347, #1348, #1349 Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- .changeset/eql-encrypted-json-target.md | 5 + docs/plans/2026-10-17-json-tree-index.md | 2 +- languages/golang/encrypt/guest/src/targets.rs | 14 +- .../golang/encrypt/guest/tests/native_ops.rs | 3 +- packages/eql/crates/eql-bindings/CHANGELOG.md | 12 + .../eql-bindings/src/encryption/targets.rs | 121 +++++- .../crates/eql-bindings/src/v4/inventory.rs | 9 +- .../eql/crates/eql-bindings/src/v4/targets.rs | 17 + .../crates/eql-bindings/tests/v4_inventory.rs | 15 +- .../eql/crates/eql-codegen/src/bindings.rs | 32 +- packages/eql/crates/eql-codegen/src/go_eql.rs | 74 +++- .../eql/crates/eql-codegen/src/targets.rs | 61 ++- packages/eql/crates/eql-domains/src/lib.rs | 103 ++++++ packages/eql/crates/eql-domains/src/tests.rs | 59 +++ .../encryption/tests/json_tree_data_plan.rs | 336 +++++++++++++++++ .../eql/tests/encryption/tests/targets.rs | 22 +- .../tests/encryption/tests/typed_queries.rs | 3 +- .../eql/tests/encryption/tests/v4_types.rs | 8 +- packages/stack-encrypt/CHANGELOG.md | 28 ++ packages/stack-encrypt/src/codes.rs | 5 + packages/stack-encrypt/src/dynamic/json.rs | 143 +++++++ packages/stack-encrypt/src/dynamic/kind.rs | 95 ++++- packages/stack-encrypt/src/dynamic/mod.rs | 31 +- packages/stack-encrypt/src/dynamic/query.rs | 2 + packages/stack-encrypt/src/dynamic/record.rs | 265 ++++++++++++- packages/stack-encrypt/src/dynamic/target.rs | 55 ++- packages/stack-encrypt/src/json/mod.rs | 13 + packages/stack-encrypt/src/json/plain.rs | 2 +- packages/stack-encrypt/src/json/text.rs | 348 ++++++++++++++++++ 29 files changed, 1828 insertions(+), 55 deletions(-) create mode 100644 .changeset/eql-encrypted-json-target.md create mode 100644 packages/eql/tests/encryption/tests/json_tree_data_plan.rs create mode 100644 packages/stack-encrypt/src/dynamic/json.rs create mode 100644 packages/stack-encrypt/src/json/text.rs diff --git a/.changeset/eql-encrypted-json-target.md b/.changeset/eql-encrypted-json-target.md new file mode 100644 index 000000000..71e8436bb --- /dev/null +++ b/.changeset/eql-encrypted-json-target.md @@ -0,0 +1,5 @@ +--- +"@cipherstash/eql": minor +--- + +**The `json_tree` family is in the EQL catalog, and a data plan can produce `public.eql_v4_json_tree` by name: the target `EncryptedJson`.** A language SDK names it in a plan field with the `json` type, gives it the JSON index's options (`{"name": "EncryptedJson", "options": {"sealed_root": true, "ordered": ["$.age"]}}`, or the bare name for the defaults), and hands each document over as UTF-8 JSON text; the `eql-bindings` crate's by-name dispatch (`encryption::targets::encrypt_with`, `decrypt`) seals it through the same JSON index layout typed Rust uses and opens it back to its text. The text is parsed by Stack Encrypt's rules, before any key is requested: an object that names a member twice, an integer literal outside the 64-bit range, and text that is not JSON are refused (`TargetError::Json`, with a code and never a byte of the text). The catalog lists the family once (`eql_domains::DOCUMENT_FAMILIES`), so the target table, `v4::all()` / `v4::all_query()` and the Go types (`eql.EncryptedJSON`, `EncryptedJSONQuery`, `EncryptedJSONLeafQuery`, `EncryptedJSONEntry`) are generated from it. No SQL changes. diff --git a/docs/plans/2026-10-17-json-tree-index.md b/docs/plans/2026-10-17-json-tree-index.md index f8d869b20..9f8065607 100644 --- a/docs/plans/2026-10-17-json-tree-index.md +++ b/docs/plans/2026-10-17-json-tree-index.md @@ -369,7 +369,7 @@ Each phase is a PR against `next`, where #1209 landed. The index, the SQL and the live oracle landed in the pull request stacked on this document; what follows records where the code is and where it departs from the design above, so the two can be read against each other. Phase 0 was folded into Phase 2, and Phases 3 and 4 remain open. -**Where it is.** The engine half is `packages/stack-encrypt/src/json/`: `Json` is the index (`Index`, term `JsonTerms`), `target::json_document` the sealing operation, `JsonDocument` the stored term and an `EncryptFrom` / `DecryptInto` target, `JsonQuery` the needle for `Contains`, `ContainedBy` and `AtPath`, `JsonLeafQuery` the operand for the orderings at a path, and `JsonSelector` / `JsonSubtreeKey` the extraction keys, derived through `KeysetCipher::json_selector` and `json_subtree_key`. `IndexSpec::Json(JsonOptions)` is the data form, with the key `"json"` and `{"json": {"sealed_root": …}}` as its wire form; a data plan reads it and refuses to run it (`Reason::JsonIndexPending`). The SQL is `packages/eql/src/v3/json_tree/`, confined to EQL v4; the EQL types are `packages/eql/crates/eql-bindings/src/v4/json_tree.rs`; the oracle is `packages/eql/tests/encryption/tests/json_tree.rs`, which stores the design's corpus, and batches of documents proptest generates (`json_tree/generated.rs`), encrypted beside their plaintexts and checks that `@>` and `<@` over the encrypted column select the rows PostgreSQL's own `jsonb` operators select over the plaintext column, on the GIN index and on a sequential scan, each plan shown in `EXPLAIN`. The pure-Rust half of that oracle is `packages/stack-encrypt/tests/json_index.rs` (`given_the_design_corpus` and the proptest properties in `given_generated_documents`), against a model of `jsonb @>` pinned to the answers in `jsonb-semantics.sql`. +**Where it is.** The engine half is `packages/stack-encrypt/src/json/`: `Json` is the index (`Index`, term `JsonTerms`), `target::json_document` the sealing operation, `JsonDocument` the stored term and an `EncryptFrom` / `DecryptInto` target, `JsonQuery` the needle for `Contains`, `ContainedBy` and `AtPath`, `JsonLeafQuery` the operand for the orderings at a path, and `JsonSelector` / `JsonSubtreeKey` the extraction keys, derived through `KeysetCipher::json_selector` and `json_subtree_key`. `IndexSpec::Json(JsonOptions)` is the data form, with the key `"json"` and `{"json": {"sealed_root": …}}` as its wire form; a data plan refuses it as an output (`Reason::JsonIndexPending`) and reaches it through the `EncryptedJson` target, whose options are the index's (`TargetOptions::Json`), on a field of the `json` kind. The SQL is `packages/eql/src/v3/json_tree/`, confined to EQL v4; the EQL types are `packages/eql/crates/eql-bindings/src/v4/json_tree.rs`; the oracle is `packages/eql/tests/encryption/tests/json_tree.rs`, which stores the design's corpus, and batches of documents proptest generates (`json_tree/generated.rs`), encrypted beside their plaintexts and checks that `@>` and `<@` over the encrypted column select the rows PostgreSQL's own `jsonb` operators select over the plaintext column, on the GIN index and on a sequential scan, each plan shown in `EXPLAIN`. The pure-Rust half of that oracle is `packages/stack-encrypt/tests/json_index.rs` (`given_the_design_corpus` and the proptest properties in `given_generated_documents`), against a model of `jsonb @>` pinned to the answers in `jsonb-semantics.sql`. **Decisions taken by the code.** Of the eleven listed below: 1, hex for every sixteen-byte field and base64 for `c` and `h`, pending Phase 3 (taken on 2026-10-23, Phase 3d: base64 for every field the SQL reads, base85 for `c`); 2, leaf ordering terms bound to the column only, numbers as the `f64` they round to and strings as text, under the scalar `Ope` index's derivation; 3, wildcard arrays only; 4, group tags on, and only on container array elements; 5, the child count is in every container node's ciphertext, and a truncated subtree is `OpenedType`, not a shorter array; 7, plain SHA-256; 8, the option is the boolean `sealed_root`, on by default, recorded in the header's flags; 9, the projection is built from the tags server-side (`eql_v4.json_tree_query`) and `groups_ok` was not needed; 10, position-keyed parent tags on, the ordinal stored as `o`, object members ranked under the parent's token; 11, keyed: in this first form a child's token was `PRF(json/token-step ‖ column, PAE(t(parent), 0x01, key))` (`0x02, LE64(index)` for an element) and an object member's rank `PRF(json/rank ‖ column, PAE(t(parent), key))`, each under its own PAE domain, so a token's holder could neither test key names against the child tokens the walk unwraps nor rank guessed names against the ordinals. Decision 6 is out of scope, as proposed. (Both steps were first hashed unkeyed, as the token chain section above writes them, and were corrected before any release.) diff --git a/languages/golang/encrypt/guest/src/targets.rs b/languages/golang/encrypt/guest/src/targets.rs index 9d72c6615..d32ab609a 100644 --- a/languages/golang/encrypt/guest/src/targets.rs +++ b/languages/golang/encrypt/guest/src/targets.rs @@ -58,11 +58,13 @@ impl TargetResolver for EqlTargets { fn encrypt<'a, K: 'static>( &self, name: &str, + options: Option<&stack_encrypt::dynamic::TargetOptions>, keyset: &'a stack_encrypt::KeysetCipher<'_, K>, label: &stack_encrypt::Label, plaintext: stack_encrypt::dynamic::ValueFrame, ) -> Result, K>, TargetError> { - eql_bindings::encryption::targets::encrypt(name, keyset, label, plaintext).map_err(convert) + eql_bindings::encryption::targets::encrypt_with(name, options, keyset, label, plaintext) + .map_err(convert) } fn decrypt<'a, K: 'static>( @@ -143,6 +145,13 @@ fn convert(error: eql_bindings::encryption::targets::TargetError) -> TargetError expected: Some(expected), found, }, + // The engine's code for what was wrong with the document, never + // the document. + Eql::Json { target, source } => TargetError::Json { + name: String::new(), + target: target.to_owned(), + reason: source.code(), + }, // The parser's kind and position, never its message: serde_json // quotes the input it refused, and the input is stored ciphertext. Eql::Stored { target, source } => TargetError::Stored { @@ -239,7 +248,8 @@ mod tests { "DoubleEq", "DoubleOrdOre", "DoubleOrd", - "DoubleOrdOpe" + "DoubleOrdOpe", + "EncryptedJson" ], "the engine's producible types, in catalog order" ); diff --git a/languages/golang/encrypt/guest/tests/native_ops.rs b/languages/golang/encrypt/guest/tests/native_ops.rs index be9cb754d..e91aae234 100644 --- a/languages/golang/encrypt/guest/tests/native_ops.rs +++ b/languages/golang/encrypt/guest/tests/native_ops.rs @@ -1412,7 +1412,8 @@ fn targets_lists_what_the_build_holds() { "DoubleEq", "DoubleOrdOre", "DoubleOrd", - "DoubleOrdOpe" + "DoubleOrdOpe", + "EncryptedJson" ], "the engine's producible types, in catalog order" ); diff --git a/packages/eql/crates/eql-bindings/CHANGELOG.md b/packages/eql/crates/eql-bindings/CHANGELOG.md index aa5882c69..355f6b817 100644 --- a/packages/eql/crates/eql-bindings/CHANGELOG.md +++ b/packages/eql/crates/eql-bindings/CHANGELOG.md @@ -9,6 +9,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- **`EncryptedJson`, the `json_tree` document, by name** (`targets` + feature, #1348, #1349). The target table has a row for it, generated + from the catalog's document family (`eql_domains::DOCUMENT_FAMILIES`): + produced from the `json` kind, carrying the `json` index, with + `EncryptedJsonQuery` (`eql_v4.query_json_tree`) as its needle. + `encryption::targets::encrypt_with` takes the field's `TargetOptions` + and seals the document's JSON text through `v4::json_tree::JsonTree` + under them; `decrypt` opens it back to its text. A value that is not a + JSON document is `TargetError::Json`, options on a type that takes none + `TargetError::Options`. `v4::all()` lists `JsonTreeDocument` and + `v4::all_query()` lists `JsonTreeQuery` and `JsonTreeLeafQuery`. + - **The EQL v4 `json_tree` family's types, `v4::json_tree`** (`stack-encrypt` feature). `JsonTreeDocument` is `public.eql_v4_json_tree`, `{v, i, h, sv, q}`: the document's entries, one per JSON node, and `q`, its containment diff --git a/packages/eql/crates/eql-bindings/src/encryption/targets.rs b/packages/eql/crates/eql-bindings/src/encryption/targets.rs index 344315c19..4bf82085c 100644 --- a/packages/eql/crates/eql-bindings/src/encryption/targets.rs +++ b/packages/eql/crates/eql-bindings/src/encryption/targets.rs @@ -88,7 +88,8 @@ use std::fmt; use serde::{de::DeserializeOwned, Serialize}; -use stack_encrypt::dynamic::{ValueFrame, ValueKind}; +use stack_encrypt::dynamic::{TargetOptions, ValueFrame, ValueKind}; +use stack_encrypt::json::JsonOptions; use stack_encrypt::kms::MaybeSend; use stack_encrypt::target::{ExpectedContext, IndexSpec}; use stack_encrypt::{ @@ -232,6 +233,25 @@ pub enum TargetError { /// passthrough value, which no kind holds. found: Option, }, + /// The value is not a JSON document by the engine's rules, for a type + /// that stores one ([`stack_encrypt::json::parse_text`]). `code` is the + /// engine's code for what was wrong, never a byte of the value. + #[error("{target} stores a JSON document, and the value is not one: {source}")] + Json { + /// The type the value was handed to. + target: &'static str, + /// What was wrong. + #[source] + source: stack_encrypt::dynamic::JsonValueError, + }, + /// Options were given to a type that takes none: a type takes the + /// options of the index it carries, and only a type that stores a + /// JSON document carries one with options. + #[error("{name} takes no options")] + Options { + /// The type's name. + name: &'static str, + }, /// The stored bytes do not parse as the type: not JSON, not this /// domain's shape, or another EQL version. #[error("stored value is not a {target}: {source}")] @@ -314,9 +334,41 @@ pub fn encrypt<'a, K: 'static>( keyset: &'a KeysetCipher<'_, K>, context: &Label, plaintext: ValueFrame, +) -> Result, K>, TargetError> { + encrypt_with(name, None, keyset, context, plaintext) +} + +/// [`encrypt`], under the options the plan gives the field +/// ([`TargetOptions`]): a type that stores a JSON document +/// (`EncryptedJson`) seals under the JSON index's options, the defaults +/// when `options` is `None`. Every other type takes none. +/// +/// # Errors +/// +/// As [`encrypt`], [`TargetError::Options`] for options a type does not +/// take, and [`TargetError::Json`] for a value that is not a JSON document +/// by the engine's rules, for a type that stores one. +pub fn encrypt_with<'a, K: 'static>( + name: &str, + options: Option<&TargetOptions>, + keyset: &'a KeysetCipher<'_, K>, + context: &Label, + plaintext: ValueFrame, ) -> Result, K>, TargetError> { producible(name)?; let column = Identifier::from_label(context)?; + if let Some(document) = document(name) { + let options = match options { + None => JsonOptions::default(), + Some(options) => options.json().cloned().ok_or(TargetError::Options { + name: document.name, + })?, + }; + return encrypt_document(document.name, &options, keyset, column, &plaintext); + } + if let Some(target) = options.and(target(name)) { + return Err(TargetError::Options { name: target.name }); + } encrypt_named(name, keyset, column, plaintext) } @@ -338,9 +390,76 @@ pub fn decrypt<'a, K: 'static>( ) -> Result, TargetError> { producible(name)?; let column = Identifier::from_label(context)?; + if let Some(document) = document(name) { + return decrypt_document(document.name, opener.into(), column, stored); + } decrypt_named(name, opener.into(), column, stored) } +/// The stored domain of a document family (`eql-domains`' +/// `DOCUMENT_FAMILIES`), when `name` is one's target: a type the generated +/// dispatch has no arm for, whose plan is the JSON index's, under options. +fn document(name: &str) -> Option<&'static Target> { + target(name).filter(|target| crate::v4::targets::DOCUMENT_FAMILIES.contains(&target.family)) +} + +/// Seal one `EncryptedJson` value: the document the field's text holds, +/// parsed by the engine's rules, laid out by the JSON index under +/// `options`, under the column. The parsed document is wiped when this +/// returns; the pending has taken what it seals. +fn encrypt_document<'a, K: 'static>( + name: &'static str, + options: &JsonOptions, + keyset: &'a KeysetCipher<'_, K>, + column: NonEmpty, + plaintext: &ValueFrame, +) -> Result, K>, TargetError> { + let document = + stack_encrypt::dynamic::read_json(plaintext).map_err(|source| TargetError::Json { + target: name, + source, + })?; + Ok(keyset + .run( + stack_encrypt::target::Layout::::encryption::( + &crate::v4::json_tree::JsonTree::new(options.clone()), + ), + document.value(), + column, + ) + .try_map(|value| serde_json::to_vec(&value).map_err(|error| Error::Other(Box::new(error))))) +} + +/// Open one stored `EncryptedJson` value to the document's text, checking +/// it was stored under the column. +fn decrypt_document<'a, K: 'static>( + name: &'static str, + opener: Opener<'a, K>, + column: NonEmpty, + stored: &[u8], +) -> Result, TargetError> { + use crate::v4::json_tree::JsonTreeDocument; + let value: JsonTreeDocument = + serde_json::from_slice(stored).map_err(|source| TargetError::Stored { + target: name, + source, + })?; + let expected: ExpectedContext = column.into(); + Ok(match opener { + Opener::Client(cipher) => { + cipher.decrypt_as::(value, expected) + } + Opener::Keyset(keyset) => { + keyset.decrypt_as::(value, expected) + } + } + .try_map(|mut document| { + let frame = stack_encrypt::dynamic::json_frame(&document); + stack_encrypt::json::wipe_value(&mut document); + frame + })) +} + /// Run the named type's query twin for one plaintext: the operand that /// matches stored values of the type under `context`'s column, as JSON /// bytes. A query derives no data key, so the pending settles without I/O. diff --git a/packages/eql/crates/eql-bindings/src/v4/inventory.rs b/packages/eql/crates/eql-bindings/src/v4/inventory.rs index a8b77dad9..0793fa22f 100644 --- a/packages/eql/crates/eql-bindings/src/v4/inventory.rs +++ b/packages/eql/crates/eql-bindings/src/v4/inventory.rs @@ -3,7 +3,8 @@ use super::DomainType; use std::marker::PhantomData; /// Every v4 stored-payload domain type, in `eql-domains::CATALOG` -/// order — generated. Scalar domains only: v4 has no SteVec bindings. +/// order, then each document family's stored document — generated. +/// v4 has no SteVec bindings. pub fn all() -> Vec> { vec![ Box::new(PhantomData::), @@ -56,10 +57,12 @@ pub fn all() -> Vec> { Box::new(PhantomData::), Box::new(PhantomData::), Box::new(PhantomData::), + Box::new(PhantomData::), ] } /// Every v4 QUERY-operand twin (`eql_v4.query_`), in -/// `eql-domains::CATALOG` order — generated. +/// `eql-domains::CATALOG` order, then each document family's +/// containment needle and ordering operand — generated. pub fn all_query() -> Vec> { vec![ Box::new(PhantomData::), @@ -101,6 +104,8 @@ pub fn all_query() -> Vec> { Box::new(PhantomData::), Box::new(PhantomData::), Box::new(PhantomData::), + Box::new(PhantomData::), + Box::new(PhantomData::), ] } /// Every v4 RANGE query operand (`eql_v4.query__range_`), diff --git a/packages/eql/crates/eql-bindings/src/v4/targets.rs b/packages/eql/crates/eql-bindings/src/v4/targets.rs index 1029d63c7..793fcffed 100644 --- a/packages/eql/crates/eql-bindings/src/v4/targets.rs +++ b/packages/eql/crates/eql-bindings/src/v4/targets.rs @@ -630,7 +630,24 @@ pub const TARGETS: &[Target] = &[ "a JSON value seals as a tree of leaves and EQL's `c` holds one; how a document becomes one sealed leaf is not decided", ), }, + Target { + name: "EncryptedJson", + family: "json_tree", + suffix: "", + plaintext: Some("json"), + sql_domain: "public.eql_v4_json_tree", + indexes: &["json"], + query: Some("EncryptedJsonQuery"), + query_sql_domain: Some("eql_v4.query_json_tree"), + producible: true, + reason: None, + }, ]; +/// The families whose stored type stores a document +/// (`eql-domains::DOCUMENT_FAMILIES`): the dispatch below has no +/// arm for them, and `crate::encryption::targets` runs their types +/// under the field's options. +pub(crate) const DOCUMENT_FAMILIES: &[&str] = &["json_tree"]; /// Run the named type's own encryption plan for one field, or refuse /// the name: the type is not producible (`TargetError::Unproducible`) /// or does not exist (`TargetError::Unknown`). diff --git a/packages/eql/crates/eql-bindings/tests/v4_inventory.rs b/packages/eql/crates/eql-bindings/tests/v4_inventory.rs index 31bbb2471..c0b588a95 100644 --- a/packages/eql/crates/eql-bindings/tests/v4_inventory.rs +++ b/packages/eql/crates/eql-bindings/tests/v4_inventory.rs @@ -10,10 +10,18 @@ use eql_bindings::v4; use eql_domains::{Term, CATALOG}; use serde_json::{json, Value}; -/// The catalog's scalar domains under their v4 names, in catalog order. v4 has -/// no SteVec bindings, so only scalar domains appear. The catalog spells each -/// name with the `eql_v3_` prefix, which is swapped for v4's. +/// The catalog's scalar domains under their v4 names, in catalog order, then +/// each document family's stored document. v4 has no SteVec bindings, so +/// only scalar domains appear from `CATALOG`. The catalog spells each name +/// with the `eql_v3_` prefix, which is swapped for v4's; a document family +/// is v4 only and spells its own. fn expected_v4_domains() -> Vec { + let documents = eql_domains::DOCUMENT_FAMILIES.iter().flat_map(|f| { + f.domains + .iter() + .filter(|d| d.role == eql_domains::DocumentRole::Stored) + .map(|d| d.sql_domain.trim_start_matches("public.").to_owned()) + }); CATALOG .iter() .flat_map(|spec| { @@ -28,6 +36,7 @@ fn expected_v4_domains() -> Vec { .unwrap_or_else(|| panic!("{name} does not carry the catalog typname prefix")); format!("{}{bare}", v4::PUBLIC_TYPNAME_PREFIX) }) + .chain(documents) .collect() } diff --git a/packages/eql/crates/eql-codegen/src/bindings.rs b/packages/eql/crates/eql-codegen/src/bindings.rs index add9714c8..32aa51d33 100644 --- a/packages/eql/crates/eql-codegen/src/bindings.rs +++ b/packages/eql/crates/eql-codegen/src/bindings.rs @@ -1060,6 +1060,32 @@ pub fn render_v4_inventory_rs() -> String { quote! { Box::new(PhantomData::), } }) .collect(); + // The document families' hand-written types: the stored document in + // `all()`, its two query operands in `all_query()`. The entry `->` + // returns is neither a column nor an operand, so it is in neither. + let document = |roles: &'static [eql_domains::DocumentRole]| -> TokenStream { + eql_domains::DOCUMENT_FAMILIES + .iter() + .flat_map(|f| f.domains.iter().map(move |d| (f, d))) + .filter(|(_, d)| roles.contains(&d.role)) + .map(|(f, d)| { + let m = format_ident!("{}", f.name); + let s = format_ident!("{}", d.rust); + quote! { Box::new(PhantomData::), } + }) + .collect() + }; + let all_entries = { + let documents = document(&[eql_domains::DocumentRole::Stored]); + quote! { #all_entries #documents } + }; + let query_entries = { + let documents = document(&[ + eql_domains::DocumentRole::Query, + eql_domains::DocumentRole::LeafQuery, + ]); + quote! { #query_entries #documents } + }; let range_entries: TokenStream = eql_domains::families_with_scalar_domains() .flat_map(|f| f.range_terms().into_iter().map(move |t| (f, t))) .map(|(f, t)| { @@ -1081,7 +1107,8 @@ pub fn render_v4_inventory_rs() -> String { use super::DomainType; /// Every v4 stored-payload domain type, in `eql-domains::CATALOG` - /// order — generated. Scalar domains only: v4 has no SteVec bindings. + /// order, then each document family's stored document — generated. + /// v4 has no SteVec bindings. pub fn all() -> Vec> { vec![ #all_entries @@ -1089,7 +1116,8 @@ pub fn render_v4_inventory_rs() -> String { } /// Every v4 QUERY-operand twin (`eql_v4.query_`), in - /// `eql-domains::CATALOG` order — generated. + /// `eql-domains::CATALOG` order, then each document family's + /// containment needle and ordering operand — generated. pub fn all_query() -> Vec> { vec![ #query_entries diff --git a/packages/eql/crates/eql-codegen/src/go_eql.rs b/packages/eql/crates/eql-codegen/src/go_eql.rs index c82396a56..1a5a7c37f 100644 --- a/packages/eql/crates/eql-codegen/src/go_eql.rs +++ b/packages/eql/crates/eql-codegen/src/go_eql.rs @@ -34,12 +34,25 @@ pub const GO_EQL_PATH: &str = "../../languages/golang/encrypt/eql/eql_gen.go"; pub const GO_GENERATED_MARKER: &str = "// Code generated by eql-codegen from the eql-domains catalog. DO NOT EDIT."; -/// The Go type name of a catalog name: the same, save the JSON family. +/// The Go type name of a catalog name: the same, save that the word +/// `Json` is spelled `JSON`, as Go spells initialisms: `Json` is `JSON`, +/// `EncryptedJsonQuery` is `EncryptedJSONQuery`. `Json` is a word where +/// it ends the name or the next letter starts another (upper case). pub fn go_name(name: &str) -> String { - match name.strip_prefix("Json") { - Some(rest) => format!("JSON{rest}"), - None => name.to_owned(), + let mut out = String::with_capacity(name.len()); + let mut rest = name; + while let Some(at) = rest.find("Json") { + let after = &rest[at + 4..]; + out.push_str(&rest[..at]); + if after.chars().next().is_none_or(char::is_uppercase) { + out.push_str("JSON"); + } else { + out.push_str("Json"); + } + rest = after; } + out.push_str(rest); + out } /// A Go string literal. @@ -155,10 +168,38 @@ pub fn render_go_eql() -> String { methods(&mut out, &query_go, query_domain); } } + document_types(&mut out); range_types(&mut out); out } +/// The document families' types beside their stored type and its needle: +/// the ordering operand and the entry the SQL returns (`->`, the walks), +/// each the JSON bytes it is in SQL, like every other type. Not in the +/// `Types` table, which lists stored types. +fn document_types(out: &mut String) { + use eql_domains::DocumentRole; + for family in eql_domains::DOCUMENT_FAMILIES { + for domain in family.domains { + let what = match domain.role { + DocumentRole::Stored | DocumentRole::Query => continue, + DocumentRole::LeafQuery => { + "the ordering operand at a path, compared with an entry `->` extracts" + } + DocumentRole::Entry => "one node of a document, as `->` and the walks return it", + }; + let go = go_name(domain.name); + let _ = write!( + out, + "\n// {go} is the {family} family's {what}: the value {sql} holds.\ntype {go} []byte\n", + family = family.name, + sql = domain.sql_domain, + ); + methods(out, &go, domain.sql_domain); + } + } +} + /// The range query types: one per family and ordering term, the operand of /// `eql_v4.between` (ADR-0009 decision 7), so a `Between` term binds as one /// parameter. Each holds the JSON `{"v", "i", "lo", "hi"}` value as bytes, @@ -209,6 +250,31 @@ mod tests { assert_eq!(go_name("TextEqQuery"), "TextEqQuery"); assert_eq!(go_name("Json"), "JSON"); assert_eq!(go_name("SteVecDocument"), "SteVecDocument"); + assert_eq!(go_name("EncryptedJson"), "EncryptedJSON"); + assert_eq!(go_name("EncryptedJsonLeafQuery"), "EncryptedJSONLeafQuery"); + assert_eq!(go_name("Jsonify"), "Jsonify", "a word, not a prefix"); + } + + #[test] + fn the_document_family_has_its_go_types() { + let out = render_go_eql(); + for go in [ + "EncryptedJSON", + "EncryptedJSONQuery", + "EncryptedJSONLeafQuery", + "EncryptedJSONEntry", + ] { + assert_eq!( + out.matches(&format!("\ntype {go} []byte\n")).count(), + 1, + "{go}" + ); + } + assert!(out.contains( + "Name: \"EncryptedJson\", GoName: \"EncryptedJSON\", Family: \"json_tree\", Suffix: \"\", \ + Plaintext: \"json\", SQLDomain: \"public.eql_v4_json_tree\", Indexes: []string{\"json\"}, \ + Query: \"EncryptedJsonQuery\", QuerySQLDomain: \"eql_v4.query_json_tree\", Producible: true" + )); } #[test] diff --git a/packages/eql/crates/eql-codegen/src/targets.rs b/packages/eql/crates/eql-codegen/src/targets.rs index 0d5dca599..aa8670fd7 100644 --- a/packages/eql/crates/eql-codegen/src/targets.rs +++ b/packages/eql/crates/eql-codegen/src/targets.rs @@ -187,11 +187,45 @@ pub(crate) struct Row { /// The operators the type answers in the typed query chain: none /// unless it is producible and has a query twin ([`answers_for_indexes`]). pub(crate) answers: Vec, + /// A document family's stored type (`eql_domains::DOCUMENT_FAMILIES`): + /// its plan is the JSON index's under the field's options, so the + /// generated dispatch has no arm for it and + /// `encryption::targets` runs it by hand. + pub(crate) document: bool, } -/// Every row, in catalog order: one per stored domain. +/// Every row, in catalog order: one per stored domain, then one per +/// document family's stored domain. pub(crate) fn rows() -> Vec { - stored_payload_domains().map(|(f, d)| row(f, d)).collect() + stored_payload_domains() + .map(|(f, d)| row(f, d)) + .chain(eql_domains::DOCUMENT_FAMILIES.iter().map(document_row)) + .collect() +} + +/// A document family's row: its stored type, produced from the family's +/// plaintext kind, carrying its index, with its containment needle as the +/// query twin. Producible: `encryption::targets` runs it by hand. +fn document_row(family: &'static eql_domains::DocumentFamily) -> Row { + use eql_domains::DocumentRole; + let stored = family + .domain(DocumentRole::Stored) + .expect("a document family has a stored domain"); + Row { + name: stored.name.to_owned(), + family: family.name, + suffix: String::new(), + plaintext: Some(family.plaintext), + sql_domain: stored.sql_domain.to_owned(), + indexes: vec![family.index], + query: family + .domain(DocumentRole::Query) + .map(|query| (query.name.to_owned(), query.sql_domain.to_owned())), + projections: Vec::new(), + reason: None, + answers: Vec::new(), + document: true, + } } /// The PascalCase of a bare domain name — `TextOrdOre` is family `text`, @@ -284,6 +318,7 @@ fn row(family: &'static DomainFamily, domain: &'static Domain) -> Row { }, reason, answers, + document: false, } } @@ -413,7 +448,17 @@ fn render_targets_from(rows: &[Row]) -> String { // One arm per producible type. The plaintext Rust type is the family's; // a producible family always has one, since a derive names it. - let producible: Vec<&Row> = rows.iter().filter(|r| r.reason.is_none()).collect(); + // A document family's type is run by hand, under the field's options. + let producible: Vec<&Row> = rows + .iter() + .filter(|r| r.reason.is_none() && !r.document) + .collect(); + let mut document_families: Vec<&str> = rows + .iter() + .filter(|r| r.document) + .map(|r| r.family) + .collect(); + document_families.dedup(); let arm = |r: &Row, strukt: &str| { let name = &r.name; let module = format_ident!("{}", r.family); @@ -542,6 +587,12 @@ fn render_targets_from(rows: &[Row]) -> String { #entries ]; + /// The families whose stored type stores a document + /// (`eql-domains::DOCUMENT_FAMILIES`): the dispatch below has no + /// arm for them, and `crate::encryption::targets` runs their types + /// under the field's options. + pub(crate) const DOCUMENT_FAMILIES: &[&str] = &[#(#document_families),*]; + /// Run the named type's own encryption plan for one field, or refuse /// the name: the type is not producible (`TargetError::Unproducible`) /// or does not exist (`TargetError::Unknown`). @@ -918,7 +969,9 @@ mod tests { fn a_producible_type_with_a_twin_answers_what_its_indexes_answer() { for row in rows() { let answers: Vec<&str> = row.answers.iter().map(|a| a.name()).collect(); - if row.reason.is_some() || row.query.is_none() { + // A document's operators are the JSON index's, answered in + // the typed chain by the engine's own document target. + if row.reason.is_some() || row.query.is_none() || row.document { assert!(answers.is_empty(), "{}: answers nothing", row.name); continue; } diff --git a/packages/eql/crates/eql-domains/src/lib.rs b/packages/eql/crates/eql-domains/src/lib.rs index 9636b2222..49d21cd9c 100644 --- a/packages/eql/crates/eql-domains/src/lib.rs +++ b/packages/eql/crates/eql-domains/src/lib.rs @@ -553,6 +553,109 @@ pub const CATALOG: &[DomainFamily] = &[ INTEGER, SMALLINT, BIGINT, DATE, TIMESTAMP, NUMERIC, TEXT, BOOLEAN, REAL, DOUBLE, JSON, ]; +/// The role a domain of a [`DocumentFamily`] plays: the column type, the +/// two query operands, or the entry the SQL returns from a document. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DocumentRole { + /// The stored document: the column type, and the target a data plan + /// names. + Stored, + /// The containment needle, which equality at a path also builds. + Query, + /// The ordering operand at a path. + LeafQuery, + /// One node, as `->` and the walks return it. + Entry, +} + +/// One domain of a [`DocumentFamily`]: its role, its type name across +/// languages, the hand-written Rust struct that carries it, and its +/// PostgreSQL domain. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct DocumentDomain { + /// What the domain is for. + pub role: DocumentRole, + /// The type's name across languages: what a data plan names as a + /// target, and what eql-codegen derives the Go name from + /// (`EncryptedJson` is `eql.EncryptedJSON`). A contract identifier, + /// not the Rust struct's name. + pub name: &'static str, + /// The hand-written Rust struct in `eql_bindings::v4::`. + pub rust: &'static str, + /// The schema-qualified PostgreSQL domain. + pub sql_domain: &'static str, +} + +/// A family whose payloads are structured documents: their SQL is +/// hand-written (`src/v3/json_tree/`, under `-- ONLY EQL VERSION: 4`) and so +/// are their Rust bodies, so the scalar materializers never see them. The +/// catalog lists them so the target table, the v4 inventories and the Go +/// types are generated from one place, like every other family. EQL v4 +/// only: Stack Encrypt is the one producer, and EQL v3 has none of them. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct DocumentFamily { + /// The family name: `json_tree`, the SQL directory and the target + /// table's `family`. + pub name: &'static str, + /// The plaintext kind the stored type is produced from, by the + /// engine's `ValueKind` name: `json`, UTF-8 JSON text. + pub plaintext: &'static str, + /// The engine index the stored type carries, by its + /// `IndexSpec::key()`: `json`. + pub index: &'static str, + /// The domains, the stored one first. + pub domains: &'static [DocumentDomain], +} + +impl DocumentFamily { + /// The domain playing `role`. + pub fn domain(&self, role: DocumentRole) -> Option<&'static DocumentDomain> { + self.domains.iter().find(|d| d.role == role) + } +} + +/// `json_tree`: an encrypted JSON document stored as one entry per node, +/// with exact containment, equality and ordering at paths, `->` extraction +/// and subtree walks (`docs/plans/2026-10-17-json-tree-index.md`). The +/// stored type is named for its role, `EncryptedJson`, over the SQL domain +/// `public.eql_v4_json_tree`; the Rust struct keeps its family name. +pub const JSON_TREE: DocumentFamily = DocumentFamily { + name: "json_tree", + plaintext: "json", + index: "json", + domains: &[ + DocumentDomain { + role: DocumentRole::Stored, + name: "EncryptedJson", + rust: "JsonTreeDocument", + sql_domain: "public.eql_v4_json_tree", + }, + DocumentDomain { + role: DocumentRole::Query, + name: "EncryptedJsonQuery", + rust: "JsonTreeQuery", + sql_domain: "eql_v4.query_json_tree", + }, + DocumentDomain { + role: DocumentRole::LeafQuery, + name: "EncryptedJsonLeafQuery", + rust: "JsonTreeLeafQuery", + sql_domain: "eql_v4.query_json_tree_leaf", + }, + DocumentDomain { + role: DocumentRole::Entry, + name: "EncryptedJsonEntry", + rust: "JsonTreeEntry", + sql_domain: "public.eql_v4_json_tree_entry", + }, + ], +}; + +/// The document families, in order. Separate from [`CATALOG`], which the +/// scalar SQL and binding materializers iterate: these families have no +/// generated SQL or struct bodies, and exist in EQL v4 only. +pub const DOCUMENT_FAMILIES: &[DocumentFamily] = &[JSON_TREE]; + /// The scalar (flat) families of `CATALOG`, in order — everything except the /// mixed `json` family (which is non-scalar because it carries SteVec domains). /// The DRY entry point for the ~9 scalar-only consumers (`list-types`, the diff --git a/packages/eql/crates/eql-domains/src/tests.rs b/packages/eql/crates/eql-domains/src/tests.rs index d938d62ea..df0de542e 100644 --- a/packages/eql/crates/eql-domains/src/tests.rs +++ b/packages/eql/crates/eql-domains/src/tests.rs @@ -1612,3 +1612,62 @@ mod shape_tests { assert_eq!(ope.pattern(), "^(00|01)([0-9a-f]{16})+$"); } } + +mod document_families { + use crate::{DocumentRole, DOCUMENT_FAMILIES, JSON_TREE}; + + /// Every document family has one domain per role, the stored one + /// first, each a v4 domain under the schema its role lives in, and no + /// two families share a name or a domain. + #[test] + fn each_family_has_one_domain_per_role_in_v4() { + let mut names = Vec::new(); + for family in DOCUMENT_FAMILIES { + assert_eq!( + family.domains[0].role, + DocumentRole::Stored, + "{}", + family.name + ); + for role in [ + DocumentRole::Stored, + DocumentRole::Query, + DocumentRole::LeafQuery, + DocumentRole::Entry, + ] { + assert_eq!( + family.domains.iter().filter(|d| d.role == role).count(), + 1, + "{}: {role:?}", + family.name + ); + } + for domain in family.domains { + let schema = match domain.role { + DocumentRole::Stored | DocumentRole::Entry => "public.eql_v4_", + DocumentRole::Query | DocumentRole::LeafQuery => "eql_v4.query_", + }; + assert!( + domain.sql_domain.starts_with(schema), + "{}", + domain.sql_domain + ); + names.push(domain.name); + names.push(domain.sql_domain); + } + } + let count = names.len(); + names.sort_unstable(); + names.dedup(); + assert_eq!(names.len(), count, "a name or a domain is listed twice"); + } + + #[test] + fn json_tree_is_the_encrypted_json_target() { + let stored = JSON_TREE.domain(DocumentRole::Stored).unwrap(); + assert_eq!(stored.name, "EncryptedJson"); + assert_eq!(stored.sql_domain, "public.eql_v4_json_tree"); + assert_eq!(JSON_TREE.plaintext, "json"); + assert_eq!(JSON_TREE.index, "json"); + } +} diff --git a/packages/eql/tests/encryption/tests/json_tree_data_plan.rs b/packages/eql/tests/encryption/tests/json_tree_data_plan.rs new file mode 100644 index 000000000..0b348cb9a --- /dev/null +++ b/packages/eql/tests/encryption/tests/json_tree_data_plan.rs @@ -0,0 +1,336 @@ +//! An encrypted JSON document through a data plan: the path a language SDK +//! takes (the Go guest installs the same resolver), against the typed Rust +//! path it must agree with. +//! +//! A data-plan field names the target `EncryptedJson`, declares the `json` +//! type, and hands its document over as JSON text. The engine parses it by +//! one set of rules, lays it out with the JSON index under the field's +//! options, and stores the `public.eql_v4_json_tree` value; opening gives +//! the document's text back. Here: +//! +//! - a batch of documents seals in one key request and opens to the same +//! values, scalar roots, arrays and a JSON `null` included; +//! - the data plan and `JsonTree::new(options)` write the same search terms +//! (selectors, ordering terms, the containment projection) for the same +//! document under the same column, and each opens what the other sealed; +//! - text that is not a document is refused before any key is requested, +//! naming the field and a code, never a byte of the text. + +mod common; + +use std::collections::BTreeMap; + +use eql_bindings::encryption::targets::{self as eql_targets, Target}; +use eql_bindings::v4::json_tree::{JsonTree, JsonTreeDocument}; +use eql_bindings::Identifier; +use serde_json::{json, Value}; +use stack_encrypt::dynamic::{ + self, record, Scope, TargetDescriptor, TargetError, TargetResolver, ValueFrame, +}; +use stack_encrypt::json::JsonOptions; +use stack_encrypt::target::{ExpectedContext, Layout}; +use stack_encrypt::{CipherText, Encrypt, KeysetCipher, Label, NonEmpty, Pending, StackCipher}; + +/// eql-bindings' table as the engine's resolver, as the Go guest installs +/// it (`languages/golang/encrypt/guest/src/targets.rs`). +struct Eql; + +fn describe(target: &Target) -> TargetDescriptor { + TargetDescriptor::new( + target.name, + target.family, + target.suffix, + target.plaintext_kind(), + target.sql_domain, + target.indexes.iter().map(|key| (*key).to_owned()).collect(), + target.query.map(str::to_owned), + target.query_sql_domain.map(str::to_owned), + target.producible, + target.reason.map(str::to_owned), + ) +} + +/// As the guest converts: a document refusal keeps its code. +fn convert(error: eql_targets::TargetError) -> TargetError { + match error { + eql_targets::TargetError::Json { target, source } => TargetError::Json { + name: String::new(), + target: target.to_owned(), + reason: source.code(), + }, + other => TargetError::Other(Box::new(other)), + } +} + +impl TargetResolver for Eql { + fn targets(&self) -> Vec { + eql_targets::targets().iter().map(describe).collect() + } + fn encrypt<'a, K: 'static>( + &self, + name: &str, + options: Option<&dynamic::TargetOptions>, + keyset: &'a KeysetCipher<'_, K>, + label: &Label, + plaintext: ValueFrame, + ) -> Result, K>, TargetError> { + eql_targets::encrypt_with(name, options, keyset, label, plaintext).map_err(convert) + } + fn decrypt<'a, K: 'static>( + &self, + name: &str, + cipher: &'a StackCipher, + label: &Label, + stored: &[u8], + ) -> Result, TargetError> { + eql_targets::decrypt(name, cipher, label, stored).map_err(convert) + } + fn query<'a, K: 'static>( + &self, + name: &str, + index: &stack_encrypt::target::IndexSpec, + keyset: &'a KeysetCipher<'_, K>, + label: &Label, + plaintext: ValueFrame, + ) -> Result, K>, TargetError> { + eql_targets::query(name, index, keyset, label, plaintext).map_err(convert) + } +} + +/// The options the documents are sealed under on both paths. +fn options() -> JsonOptions { + JsonOptions::default() + .ordering(["$.age", "$.visits[*].at"]) + .unwrap() +} + +/// A plan with one `json` field, `profile`, under `users/profile`, naming +/// `EncryptedJson` with [`options`], as a binding writes it. +fn plan() -> record::Plan { + let wire = json!({ + "profile": { + "context": ["users", "profile"], + "type": "json", + "null": "sql", + "target": { + "name": "EncryptedJson", + "options": {"ordered": ["$.age", "$.visits[*].at"]} + } + } + }); + record::plan_with(&dynamic::frame::serde::to_vec(&wire).unwrap(), &Eql).unwrap() +} + +fn frame(value: T) -> ValueFrame { + ValueFrame::of(value).unwrap() +} + +/// The documents a batch carries, as values. +fn documents() -> Vec { + vec![ + json!({"name": "Alice", "age": 34, "visits": [{"at": 3, "where": "x"}, {"at": 1}], "tags": ["a", "b"]}), + json!({"name": "Bob", "age": 7.5, "nested": {"deep": [[1, 2], {}]}, "none": null}), + json!([1, "two", {"three": 3}]), + json!("a scalar root"), + json!(18446744073709551615u64), + json!(-9223372036854775808i64), + json!(-926_809.505_100_808_3), + Value::Null, + json!({}), + json!([]), + ] +} + +/// The EQL value a sealed row stores for `profile`, parsed. +fn stored(row: &stack_encrypt::StackCipherText) -> JsonTreeDocument { + let CipherText::Map(fields) = row else { + panic!("a row is a map") + }; + let (_, CipherText::Map(outputs)) = fields + .iter() + .find(|(name, _)| name == "profile") + .expect("profile is stored") + else { + panic!("profile is a map of outputs"); + }; + let Some((_, CipherText::Passthrough(payload))) = outputs.iter().find(|(key, _)| key == "eql") + else { + panic!("profile's EQL value is a passthrough"); + }; + let bytes: Vec = + match dynamic::frame::serde::from_slice::(&payload.0) { + Ok(bytes) => bytes.0, + Err(_) => panic!("the EQL value is bytes"), + }; + serde_json::from_slice(&bytes).expect("the EQL value is a json_tree document") +} + +/// A byte string read out of a frame. +mod serde_bytes_like { + pub struct Bytes(pub Vec); + impl<'de> serde::Deserialize<'de> for Bytes { + fn deserialize>(d: D) -> Result { + struct V; + impl serde::de::Visitor<'_> for V { + type Value = Bytes; + fn expecting(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("bytes") + } + fn visit_bytes(self, v: &[u8]) -> Result { + Ok(Bytes(v.to_vec())) + } + fn visit_byte_buf(self, v: Vec) -> Result { + Ok(Bytes(v)) + } + } + d.deserialize_bytes(V) + } + } +} + +fn column() -> NonEmpty { + Identifier::for_column("users", "profile").unwrap() +} + +/// Every document of a batch, as JSON text, seals in one key request and +/// opens to the same value: the text is not promised back byte for byte, +/// the value is. +#[tokio::test] +async fn a_batch_of_documents_round_trips_in_one_request() { + let (cipher, calls) = common::cipher().await; + let keyset = cipher.default_keyset(); + let plan = plan(); + let texts: Vec = documents().iter().map(Value::to_string).collect(); + let source: Vec> = texts + .iter() + .map(|text| BTreeMap::from([("profile", text.as_str())])) + .collect(); + let sealed = record::encrypt_with(&keyset, frame(&source), &plan, &Eql) + .unwrap() + .await + .unwrap(); + assert_eq!( + calls.lock().unwrap().generate.len(), + 1, + "every document's key in one request" + ); + let opened = record::decrypt_with(Scope::Client(&cipher), sealed, &plan, None, &Eql) + .unwrap() + .await + .unwrap(); + assert_eq!(calls.lock().unwrap().retrieve.len(), 1, "and one to open"); + let rows: Vec> = + dynamic::frame::serde::from_slice(opened.risky_ref()).unwrap(); + let back: Vec = rows + .iter() + .map(|row| serde_json::from_slice(&row["profile"].0).unwrap()) + .collect(); + assert_eq!(back, documents()); +} + +/// The data plan and the typed layout write the same search terms for the +/// same document under the same column: every entry's selector, self and +/// parent tags' places and ordering term, and the containment projection. +/// Only the ciphertexts, the data key and the tags salted by it differ. +/// And each opens what the other sealed. +#[tokio::test] +async fn the_data_plan_and_the_typed_layout_agree() { + let (cipher, _) = common::cipher().await; + let keyset = cipher.default_keyset(); + let plan = plan(); + for document in documents() { + let text = document.to_string(); + let source = BTreeMap::from([("profile", text.as_str())]); + let sealed = record::encrypt_with(&keyset, frame(&source), &plan, &Eql) + .unwrap() + .await + .unwrap(); + let data = stored(&sealed); + let typed: JsonTreeDocument = keyset + .run( + Layout::::encryption::(&JsonTree::new(options())), + &document, + column(), + ) + .await + .unwrap(); + assert_eq!(data.i, typed.i, "{text}: the column"); + assert_eq!(data.q, typed.q, "{text}: the containment projection"); + assert_eq!(data.sv.len(), typed.sv.len(), "{text}: one entry per node"); + for (a, b) in data.sv.iter().zip(&typed.sv) { + assert_eq!(a.s, b.s, "{text}: an entry's selector"); + assert_eq!(a.op, b.op, "{text}: an entry's ordering term"); + assert_eq!(a.p.is_some(), b.p.is_some(), "{text}: a parent tag's place"); + } + + // The typed path opens the data plan's document... + let opened: Value = cipher + .decrypt_as::(data, ExpectedContext::from(column())) + .await + .unwrap(); + assert_eq!(opened, document); + // ...and the data plan opens the typed path's. + let record = CipherText::Map(vec![( + "profile".to_owned(), + CipherText::Map(vec![( + "eql".to_owned(), + CipherText::Passthrough(stack_encrypt::PassthroughFrame( + frame(stack_encrypt::Protected::new( + serde_json::to_vec(&typed).unwrap(), + )) + .risky_ref() + .to_vec(), + )), + )]), + )]); + let opened = record::decrypt_with(Scope::Client(&cipher), record, &plan, None, &Eql) + .unwrap() + .await + .unwrap(); + let row: BTreeMap = + dynamic::frame::serde::from_slice(opened.risky_ref()).unwrap(); + assert_eq!( + serde_json::from_slice::(&row["profile"].0).unwrap(), + document + ); + } +} + +/// Text that is not a document is refused when the record is read, before +/// any key is requested, naming the field and what was wrong by its code. +/// A frame null is not a JSON null: under `"null": "sql"` it stores +/// nothing. +#[tokio::test] +async fn text_that_is_not_a_document_is_refused_before_any_key() { + let (cipher, calls) = common::cipher().await; + let keyset = cipher.default_keyset(); + let plan = plan(); + for (text, code) in [ + (r#"{"secret": 1, "secret": 2}"#, "json_duplicate_key"), + ("99999999999999999999", "json_integer_range"), + (r#"{"secret": "#, "json_syntax"), + ("", "json_syntax"), + ] { + let source = BTreeMap::from([("profile", text)]); + let error = match record::encrypt_with(&keyset, frame(&source), &plan, &Eql) { + Err(error) => error, + Ok(_) => panic!("{text:?} was accepted"), + }; + let shown = format!("{error} {:?}", stack_encrypt::ErrorPayload::payload(&error)); + assert!( + matches!(&error, dynamic::Error::Target(TargetError::Json { name, reason, .. }) if name == "profile" && *reason == code), + "{text:?}: {error:?}" + ); + assert!( + !shown.contains("secret") && !shown.contains("9999"), + "{shown}" + ); + } + // A value of another kind is not a document either. + let source = BTreeMap::from([("profile", 7u32)]); + assert!(record::encrypt_with(&keyset, frame(&source), &plan, &Eql).is_err()); + assert!( + calls.lock().unwrap().generate.is_empty(), + "no key was requested" + ); +} diff --git a/packages/eql/tests/encryption/tests/targets.rs b/packages/eql/tests/encryption/tests/targets.rs index 5b2a6dd8b..c6e20053b 100644 --- a/packages/eql/tests/encryption/tests/targets.rs +++ b/packages/eql/tests/encryption/tests/targets.rs @@ -476,8 +476,25 @@ mod the_table { #[test] fn names_every_stored_catalog_domain_in_order() { - let table = targets::targets(); + let all = targets::targets(); let expected = stored_domains(); + // The document families' stored types follow the catalog's, one + // each, named as their contract names, with their plaintext kind, + // their index and their needle. + let (table, documents) = all.split_at(expected.len()); + assert_eq!(documents.len(), eql_domains::DOCUMENT_FAMILIES.len()); + for (row, family) in documents.iter().zip(eql_domains::DOCUMENT_FAMILIES) { + let stored = family.domain(eql_domains::DocumentRole::Stored).unwrap(); + let query = family.domain(eql_domains::DocumentRole::Query).unwrap(); + assert_eq!(row.name, stored.name); + assert_eq!(row.family, family.name); + assert_eq!(row.sql_domain, stored.sql_domain); + assert_eq!(row.plaintext, Some(family.plaintext)); + assert_eq!(row.indexes, [family.index]); + assert_eq!(row.query, Some(query.name)); + assert_eq!(row.query_sql_domain, Some(query.sql_domain)); + assert!(row.producible && row.reason.is_none()); + } assert_eq!( table.iter().map(|t| t.name).collect::>(), expected @@ -660,7 +677,8 @@ mod the_table { "DoubleEq", "DoubleOrdOre", "DoubleOrd", - "DoubleOrdOpe" + "DoubleOrdOpe", + "EncryptedJson" ], ); for row in targets::targets().iter().filter(|t| !t.producible) { diff --git a/packages/eql/tests/encryption/tests/typed_queries.rs b/packages/eql/tests/encryption/tests/typed_queries.rs index f0f2223a3..b29d939d5 100644 --- a/packages/eql/tests/encryption/tests/typed_queries.rs +++ b/packages/eql/tests/encryption/tests/typed_queries.rs @@ -72,11 +72,12 @@ impl TargetResolver for Eql { fn encrypt<'a, K: 'static>( &self, name: &str, + options: Option<&dynamic::TargetOptions>, keyset: &'a KeysetCipher<'_, K>, label: &Label, plaintext: ValueFrame, ) -> Result, K>, TargetError> { - eql_targets::encrypt(name, keyset, label, plaintext).map_err(convert) + eql_targets::encrypt_with(name, options, keyset, label, plaintext).map_err(convert) } fn decrypt<'a, K: 'static>( &self, diff --git a/packages/eql/tests/encryption/tests/v4_types.rs b/packages/eql/tests/encryption/tests/v4_types.rs index ed823cf1c..ca1c86395 100644 --- a/packages/eql/tests/encryption/tests/v4_types.rs +++ b/packages/eql/tests/encryption/tests/v4_types.rs @@ -410,7 +410,13 @@ async fn a_leaf_sealed_from_a_frame_opens_as_the_typed_plaintext() { async fn every_producible_type_dispatches_by_name() { let (cipher, _) = common::cipher().await; let keyset = cipher.default_keyset(); - let producible: Vec<_> = targets::targets().iter().filter(|t| t.producible).collect(); + // A type that stores a JSON document takes JSON text and answers the + // JSON operators, not one query per index: tests/json_tree_data_plan.rs + // runs it by name. + let producible: Vec<_> = targets::targets() + .iter() + .filter(|t| t.producible && t.plaintext != Some("json")) + .collect(); assert!(!producible.is_empty()); for target in producible { let name = target.name; diff --git a/packages/stack-encrypt/CHANGELOG.md b/packages/stack-encrypt/CHANGELOG.md index 14c92a57a..e7f4a9614 100644 --- a/packages/stack-encrypt/CHANGELOG.md +++ b/packages/stack-encrypt/CHANGELOG.md @@ -7,6 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Breaking + +- **`TargetResolver::encrypt` takes the field's target options** (#1349): + `encrypt(name, options: Option<&TargetOptions>, keyset, label, + plaintext)`. A resolver implemented outside this crate adds the + parameter; one that resolves no document type ignores it. + ### Fixed - **A float in a JSON index document opens to the value that was sealed** @@ -24,6 +31,27 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- **A JSON document through a data plan** (#1349). A plan field may + declare the new `json` value kind and name a target that stores a + document, `EncryptedJson`, with the JSON index's options: + `"target": {"name": "EncryptedJson", "options": {"sealed_root": true, + "ordered": ["$.age"]}}`, or the bare name for the defaults. The plan + resolves the options once when it is built (`FieldPlan::target_options`, + `TargetOptions::Json`), so writes, queries and reads of the field run + under one definition, and refuses options on a target that takes none + (`Reason::TargetOptions`), a `json` field with no target + (`Reason::JsonNeedsTarget`) and a sealed null on one. The document + crosses as UTF-8 JSON text in a text or bytes leaf and opens back to + its text in a bytes leaf (`dynamic::read_json`, `dynamic::json_frame`). + `json::parse_text` is the one parser: it refuses an object that names a + member twice and an integer literal outside `i64::MIN ..= u64::MAX` + (which serde_json would read as the nearest `f64`), as well as text + that is not JSON, each with a code (`JsonTextError::code`) and a + position, never a byte of the text. `TargetError::Json` carries the code + when a resolver refuses a document. `json::wipe_value` wipes a parsed + document in place. `ValueKind::holds` says whether a value of one kind + is a value of a declared one (`json` holds text and bytes). + - **A JSON index, `target::Json`** (`docs/plans/2026-10-17-json-tree-index.md`, ADR-0009). An `Index` whose stored form, `json::JsonDocument`, is one entry per node under one data key per diff --git a/packages/stack-encrypt/src/codes.rs b/packages/stack-encrypt/src/codes.rs index 9a581fb28..555a93fce 100644 --- a/packages/stack-encrypt/src/codes.rs +++ b/packages/stack-encrypt/src/codes.rs @@ -249,6 +249,11 @@ mod tests { target: target(), reason: "not JSON".into(), }, + TargetError::Json { .. } => TargetError::Json { + name: name(), + target: target(), + reason: "json_duplicate_key", + }, TargetError::Other(_) => TargetError::Other(Box::new(std::io::Error::other("boom"))), ])); errors diff --git a/packages/stack-encrypt/src/dynamic/json.rs b/packages/stack-encrypt/src/dynamic/json.rs new file mode 100644 index 000000000..2c5063b91 --- /dev/null +++ b/packages/stack-encrypt/src/dynamic/json.rs @@ -0,0 +1,143 @@ +//! A JSON document as a binding hands one over and gets one back. +//! +//! A field of kind [`json`](ValueKind::Json) carries its document as UTF-8 +//! JSON text, in a text leaf or a bytes leaf of the transport frame. The +//! engine parses it once, by [`parse_text`]'s rules, where it is sealed, +//! queried or compared, so every language seals the same document for the +//! same text. An opened document comes back as its text in a bytes leaf. +//! +//! A document's `null` is the text `null`, a present value. A field's +//! absent value is the frame's null, which a `json` field stores as SQL +//! `NULL` (`"null": "sql"`) and never as a document. + +use serde_json::Value; +use vitaminc_aead::plaintext::{Logical, Physical}; +use vitaminc_aead::Encrypt; +use vitaminc_protected::Protected; +use zeroize::Zeroizing; + +use super::term::OneScalar; +use super::{ValueFrame, ValueKind}; +use crate::json::{parse_text, to_text, wipe_value, JsonTextError}; + +/// Why a `json` field's value is not a document. +#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)] +#[non_exhaustive] +pub enum JsonValueError { + /// The value is not a text or bytes leaf: a number, a container, a + /// null or a passthrough. + #[error("a JSON document crosses as text or bytes")] + NotText, + /// The text is not a document by the engine's rules. + #[error(transparent)] + Text(#[from] JsonTextError), +} + +impl JsonValueError { + /// The error's machine-readable code: `json_not_text`, or the text + /// error's ([`JsonTextError::code`]). + pub fn code(&self) -> &'static str { + match self { + JsonValueError::NotText => "json_not_text", + JsonValueError::Text(error) => error.code(), + } + } +} + +/// A JSON document a binding handed over, parsed. Wiped when it drops: the +/// caller holds the plaintext only as long as it reads it. +pub struct JsonPlaintext(Value); + +impl JsonPlaintext { + /// The document. + pub fn value(&self) -> &Value { + &self.0 + } +} + +impl Drop for JsonPlaintext { + fn drop(&mut self) { + wipe_value(&mut self.0); + } +} + +/// Parse a `json` field's value: the document its text or bytes leaf +/// holds. +/// +/// # Errors +/// +/// [`JsonValueError::NotText`] for a value that is not a text or bytes +/// leaf, and [`JsonValueError::Text`] for text that is not a document by +/// [`parse_text`]'s rules. +pub fn read_json(value: &ValueFrame) -> Result { + if !ValueKind::Json.holds(value.kind()) { + return Err(JsonValueError::NotText); + } + value + .visit(OneScalar::new( + |ty: Option, value: Physical<'_>| match (ty, value) { + (None, Physical::Text(text)) => Ok(parse_text(text.as_bytes())), + (None, Physical::Bytes(bytes)) => Ok(parse_text(bytes)), + _ => Err(None), + }, + )) + .map_err(|_| JsonValueError::NotText)? + .map(JsonPlaintext) + .map_err(JsonValueError::Text) +} + +/// A document as the value a `json` field opens to: its JSON text +/// ([`to_text`]), in a bytes leaf. The text is written into the frame's +/// protected buffer and the copy wiped. +/// +/// # Errors +/// +/// [`crate::Error::Other`] if the frame writer refuses the bytes, which a +/// document's text never is. +pub fn json_frame(value: &Value) -> Result { + let text = Zeroizing::new(to_text(value).map_err(|e| crate::Error::Other(Box::new(e)))?); + ValueFrame::of(Protected::new(text.to_vec())).map_err(|e| crate::Error::Other(Box::new(e))) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::dynamic::test_frame::Tree; + use serde_json::json; + + fn frame(tree: Tree) -> ValueFrame { + ValueFrame::of(&tree).unwrap() + } + + #[test] + fn a_document_is_read_from_text_or_bytes() { + let text = read_json(&frame(Tree::String(r#"{"a": [1, null]}"#.into()))).unwrap(); + assert_eq!(text.value(), &json!({"a": [1, null]})); + let bytes = read_json(&frame(Tree::Bytes(b"null".to_vec()))).unwrap(); + assert_eq!(bytes.value(), &Value::Null, "a JSON null is a document"); + } + + #[test] + fn anything_else_is_not_a_document() { + for tree in [ + Tree::Null, + Tree::UInt32(1), + Tree::Array(vec![]), + Tree::Bool(true), + ] { + assert_eq!(read_json(&frame(tree)).err(), Some(JsonValueError::NotText)); + } + let error = read_json(&frame(Tree::String(r#"{"a":1,"a":2}"#.into()))) + .err() + .unwrap(); + assert_eq!(error.code(), "json_duplicate_key"); + } + + #[test] + fn an_opened_document_is_its_text_in_bytes() { + let value = json!({"name": "Alice", "n": [1.5, -2]}); + let opened = json_frame(&value).unwrap(); + assert_eq!(opened.kind(), Some(ValueKind::Bytes)); + assert_eq!(read_json(&opened).unwrap().value(), &value); + } +} diff --git a/packages/stack-encrypt/src/dynamic/kind.rs b/packages/stack-encrypt/src/dynamic/kind.rs index 4dca167a8..5381ec49e 100644 --- a/packages/stack-encrypt/src/dynamic/kind.rs +++ b/packages/stack-encrypt/src/dynamic/kind.rs @@ -132,7 +132,7 @@ fn scalar_of(kind: ValueKind) -> Option<(Option, Physical<'static>)> { ), ValueKind::String => (None, Physical::Text("")), ValueKind::Bytes => (None, Physical::Bytes(&[])), - ValueKind::Array | ValueKind::Object => return None, + ValueKind::Array | ValueKind::Object | ValueKind::Json => return None, }) } @@ -189,7 +189,7 @@ fn scalar_of(kind: ValueKind) -> Option<(Option, Physical<'static>)> { pub fn read(kind: ValueKind, value: impl Encrypt) -> Result { let refused = || Error::bad_source(Reason::FieldType); let value = ValueFrame::of(value).map_err(|_| refused())?; - if value.kind() == Some(kind) { + if kind.holds(value.kind()) { return Ok(value); } let number = (&value) @@ -392,11 +392,19 @@ pub enum ValueKind { Array, /// `"object"`. Says nothing about the entries' kinds. Object, + /// `"json"`: a JSON document, carried as UTF-8 JSON text in a text or + /// bytes leaf, which the engine parses by one set of rules + /// ([`crate::json::parse_text`]) and opens back to as bytes. A value + /// is never of this kind ([`ValueFrame::kind`] says `string` or + /// `bytes`); a field declares it, and takes a target that stores a + /// document (`EncryptedJson`). A JSON document's `null` is the text + /// `null`, not the frame's null. + Json, } impl ValueKind { /// Every kind, in declaration order. - pub const ALL: [ValueKind; 20] = [ + pub const ALL: [ValueKind; 21] = [ ValueKind::Bool, ValueKind::Int8, ValueKind::Int16, @@ -417,13 +425,14 @@ impl ValueKind { ValueKind::Bytes, ValueKind::Array, ValueKind::Object, + ValueKind::Json, ]; /// How a declaration spells this kind: `"bool"`, `"int8"`, `"int16"`, /// `"int32"`, `"int64"`, `"int128"`, `"uint8"`, `"uint16"`, `"uint32"`, /// `"uint64"`, `"uint128"`, `"float32"`, `"float64"`, `"decimal"`, - /// `"date"`, `"timestamp"`, `"string"`, `"bytes"`, `"array"` or - /// `"object"`. Frozen wire format. + /// `"date"`, `"timestamp"`, `"string"`, `"bytes"`, `"array"`, + /// `"object"` or `"json"`. Frozen wire format. pub const fn name(self) -> &'static str { match self { ValueKind::Bool => "bool", @@ -446,6 +455,7 @@ impl ValueKind { ValueKind::Bytes => "bytes", ValueKind::Array => "array", ValueKind::Object => "object", + ValueKind::Json => "json", } } @@ -477,13 +487,29 @@ impl ValueKind { ValueKind::Timestamp => { FixedType::new(Some(Logical::Timestamp(TimeUnit::Nanos)), Width::I128) } - ValueKind::String | ValueKind::Bytes | ValueKind::Array | ValueKind::Object => { - return None - } + ValueKind::String + | ValueKind::Bytes + | ValueKind::Array + | ValueKind::Object + | ValueKind::Json => return None, }) } } +impl ValueKind { + /// Whether a value of kind `found` is a value of this declared kind: + /// the same kind, or for [`Json`](Self::Json), the text or bytes the + /// document crosses as. `None`, a value with no kind (a null, an + /// undefined, a passthrough), is a value of no kind. + pub const fn holds(self, found: Option) -> bool { + match (self, found) { + (ValueKind::Json, Some(ValueKind::String | ValueKind::Bytes)) => true, + (declared, Some(found)) => declared as u8 == found as u8, + (_, None) => false, + } + } +} + impl fmt::Display for ValueKind { /// Writes [`name`](Self::name). fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { @@ -542,7 +568,8 @@ mod tests { "string", "bytes", "array", - "object" + "object", + "json" ], "the frozen names" ); @@ -599,9 +626,53 @@ mod tests { ValueKind::Bytes => tree(Tree::Bytes(b"ab".to_vec())), ValueKind::Array => tree(Tree::Array(vec![Tree::UInt32(1)])), ValueKind::Object => tree(Tree::Object(vec![("k".to_string(), Tree::UInt32(1))])), + // A document crosses as its text: a value of kind string. + ValueKind::Json => tree(Tree::String(r#"{"k": 1}"#.into())), } } + /// Every kind whose values are of that kind: all but `json`, which a + /// field declares and a value never is (it crosses as text or bytes). + fn value_kinds() -> impl Iterator { + ValueKind::ALL + .into_iter() + .filter(|kind| *kind != ValueKind::Json) + } + + /// `json` admits no index (a document is indexed by the target that + /// stores it), holds text and bytes and nothing else, and reads either + /// as it is: the text is parsed where the document is sealed. + #[test] + fn json_holds_its_text_and_admits_no_index() { + for index in indexes() { + assert!(!admits(ValueKind::Json, &index), "{index}"); + } + assert!(ValueKind::Json.holds(Some(ValueKind::String))); + assert!(ValueKind::Json.holds(Some(ValueKind::Bytes))); + for kind in value_kinds().filter(|k| !matches!(k, ValueKind::String | ValueKind::Bytes)) { + assert!(!ValueKind::Json.holds(Some(kind)), "{kind}"); + } + assert!( + !ValueKind::Json.holds(None), + "a frame null is not a document" + ); + assert!(!ValueKind::String.holds(Some(ValueKind::Bytes))); + for kind in value_kinds() { + assert!(kind.holds(Some(kind)), "{kind}"); + assert!(!kind.holds(None), "{kind}"); + } + let text = read_frame(ValueKind::Json, sample(ValueKind::Json)).expect("text"); + assert_eq!(text.kind(), Some(ValueKind::String)); + let bytes = read_frame(ValueKind::Json, Tree::Bytes(b"[1]".to_vec())).expect("bytes"); + assert_eq!(bytes.kind(), Some(ValueKind::Bytes)); + assert!(read_frame(ValueKind::Json, Tree::UInt32(1)).is_none()); + assert_eq!( + ValueKind::Json.sealed_null(), + None, + "a document has no fixed width" + ); + } + fn indexes() -> [IndexSpec; 5] { [ IndexSpec::Equality, @@ -616,7 +687,7 @@ mod tests { /// disagree about a value of any kind, and a composite admits nothing. #[test] fn admits_agrees_with_defines_for_every_kind() { - for kind in ValueKind::ALL { + for kind in value_kinds() { for index in indexes() { assert_eq!( admits(kind, &index), @@ -653,7 +724,7 @@ mod tests { /// `vitaminc-normalize` gives the kind's values an equality domain. #[test] fn admits_equality_exactly_where_vitaminc_gives_a_domain() { - for kind in ValueKind::ALL { + for kind in value_kinds() { let has_domain = matches!( vitaminc_normalize::terms(sample(kind)).as_deref(), Ok([term]) if term.path.steps().is_empty() @@ -913,7 +984,7 @@ mod tests { #[test] fn read_returns_a_value_of_the_kind_as_it_is() { - for kind in ValueKind::ALL { + for kind in value_kinds() { let read = read_frame(kind, sample(kind)).expect("its own kind"); assert_eq!(read.kind(), Some(kind)); assert_eq!( diff --git a/packages/stack-encrypt/src/dynamic/mod.rs b/packages/stack-encrypt/src/dynamic/mod.rs index 67bcd3a04..1f82a9bb5 100644 --- a/packages/stack-encrypt/src/dynamic/mod.rs +++ b/packages/stack-encrypt/src/dynamic/mod.rs @@ -38,6 +38,8 @@ //! is the same bytes (ADR-0007). The one step that stays dynamic is //! dispatching a value whose type is known only at run time to the typed //! term operation, which [`IndexSpec`]'s `Index` impls do. +//! * [`read_json`] and [`json_frame`] — a `json` field's document, from +//! the JSON text a binding hands over and back to it. //! * [`TargetResolver`] — the EQL types a build holds, installed by the host //! that links them, so a plan field may name one as its target and the //! lowering runs that type's own plan in the same request; [`NoTargets`] @@ -86,6 +88,7 @@ //! tells a binding author that. [`Scope`] is exhaustive for a different //! reason, given on the type. mod context; +mod json; mod kind; mod query; pub mod record; @@ -98,6 +101,7 @@ mod value_frame; use std::fmt; pub use context::{borrowed, context}; +pub use json::{json_frame, read_json, JsonPlaintext, JsonValueError}; /// The value kinds, the type a plan field declares in its `"type"` key. A /// kind's names are frozen wire format; see [`admits`] and [`read`] for what /// a kind means to this engine. @@ -107,7 +111,7 @@ pub use query::{ OperatorEntry, QueryTerm, }; pub use record::{FieldPlan, NullMode, Output, Plan}; -pub use target::{NoTargets, TargetDescriptor, TargetError, TargetResolver}; +pub use target::{NoTargets, TargetDescriptor, TargetError, TargetOptions, TargetResolver}; pub use term::{term, TermBytes}; pub use value_frame::{Children, FromFrameError, Modeled, Opening, ValueFrame}; @@ -535,18 +539,29 @@ reasons! { DuplicateOutput => "duplicate_output", "an output is named twice"; /// A field names `"passthrough"` beside another output. PassthroughWithOutputs => "passthrough_with_outputs", r#""passthrough" beside another output"#; - /// `"target"` is not a non-empty string. - InvalidTarget => "invalid_target", r#""target" is not a non-empty string"#; + /// `"target"` is not a non-empty string, nor an object of a + /// non-empty `"name"` and the target's `"options"`; or the options + /// do not parse as the target's (an unknown option, a path that is + /// not one, an ordered path naming a position). + InvalidTarget => "invalid_target", r#""target" is not a non-empty string or a {"name", "options"} object the target reads"#; /// `"type"` is not a string naming a value type. UnknownType => "unknown_type", r#""type" does not name a value type"#; /// The field's declared type has no such index: match on an integer, /// equality or an order index on a boolean, any index on a composite. IndexNotAdmitted => "index_not_admitted", "the declared type has no such index"; - /// The field declares the JSON index, which a data plan names but - /// cannot run yet: the engine's typed path produces a JSON - /// document, and the binding's half, a document plaintext and its - /// EQL target, is not written. - JsonIndexPending => "json_index_pending", "the JSON index has no data-plan form yet"; + /// The field declares the JSON index as an output. A data plan + /// reaches the JSON index through a target that stores a document + /// (`EncryptedJson`, with the `json` type), whose options are the + /// index's; the index is not an output of its own. + JsonIndexPending => "json_index_pending", "the JSON index is reached through a target that stores a document, not as an output"; + /// The field declares the `json` type and names no target: a JSON + /// document is stored through a target that stores one + /// (`EncryptedJson`). + JsonNeedsTarget => "json_needs_target", r#"a "json" field names a target that stores a document"#; + /// The field gives its target options the target does not take: a + /// target takes the options of the index it carries, and this one + /// carries none that has any. + TargetOptions => "target_options", "the target takes no options"; /// Two fields are keyed under one identity. SharedIdentity => "shared_identity", "two fields are keyed under one identity"; /// The plan has no field of the name asked for. diff --git a/packages/stack-encrypt/src/dynamic/query.rs b/packages/stack-encrypt/src/dynamic/query.rs index 518fb4a46..224c49023 100644 --- a/packages/stack-encrypt/src/dynamic/query.rs +++ b/packages/stack-encrypt/src/dynamic/query.rs @@ -2444,6 +2444,7 @@ mod tests { fn encrypt<'a, K: 'static>( &self, _: &str, + _: Option<&crate::dynamic::TargetOptions>, _: &'a KeysetCipher<'_, K>, _: &Label, _: ValueFrame, @@ -2621,6 +2622,7 @@ mod tests { fn encrypt<'a, K: 'static>( &self, _: &str, + _: Option<&crate::dynamic::TargetOptions>, _: &'a KeysetCipher<'_, K>, _: &Label, _: ValueFrame, diff --git a/packages/stack-encrypt/src/dynamic/record.rs b/packages/stack-encrypt/src/dynamic/record.rs index 0e1c34b10..28bb57e25 100644 --- a/packages/stack-encrypt/src/dynamic/record.rs +++ b/packages/stack-encrypt/src/dynamic/record.rs @@ -147,7 +147,7 @@ use serde::de::{Deserializer, MapAccess, Visitor}; use serde::Deserialize; use super::term::{IndexWire, JsonWire, MatchWire}; -use super::{ValueFrame, ValueKind}; +use super::{TargetOptions, ValueFrame, ValueKind}; use vitaminc_aead::frame::{Frame, FrameWriter}; use vitaminc_aead::plaintext::{Logical, MapSink, Physical, SeqSink, Sink}; use vitaminc_aead::{DecodeError, Encrypt}; @@ -273,6 +273,7 @@ pub struct FieldPlan { extension: Vec>, outputs: Vec, target: Option, + target_options: Option, field_type: Option, null: Option, } @@ -329,6 +330,7 @@ impl FieldPlan { extension, outputs, target: None, + target_options: None, field_type: None, null: None, }) @@ -365,11 +367,32 @@ impl FieldPlan { extension, outputs: Vec::new(), target: Some(target), + target_options: None, field_type: None, null: None, }) } + /// Give the field's target its options: the options of the index the + /// target carries, such as a JSON document's sealed root and ordered + /// paths. Whether the target takes them is decided when the plan is + /// built ([`Plan::new_with`]), against the resolver; a target that + /// carries the JSON index and is given none takes the index's + /// defaults. Options on a field with no target are refused there too. + #[must_use] + pub fn with_target_options(mut self, options: TargetOptions) -> Self { + self.target_options = Some(options); + self + } + + /// The options the field's target runs under: the ones given, or for a + /// target that carries the JSON index and was given none, the index's + /// defaults, once the plan is built. `None` for a target that takes + /// none, and for a field with no target. + pub fn target_options(&self) -> Option<&TargetOptions> { + self.target_options.as_ref() + } + /// Declare the type of this field's values. /// /// # Errors @@ -744,6 +767,31 @@ impl Plan { (None, Some(kind)) => field.field_type = Some(kind), _ => {} } + // A target takes the options of the index it carries, the JSON + // index's alone today. Defaults are filled in here, once, so + // every write, query and read of the field runs under one + // resolved definition. + let carries_json = descriptor + .indexes + .iter() + .any(|index| index == IndexSpec::Json(Default::default()).key()); + field.target_options = match (field.target_options.take(), carries_json) { + (None, false) => None, + (None, true) => Some(TargetOptions::Json(Default::default())), + (Some(TargetOptions::Json(options)), true) => { + options.check().map_err(|_| Error::Plan { + field: Some(field.name.clone()), + reason: Reason::InvalidTarget, + })?; + Some(TargetOptions::Json(options)) + } + (Some(_), false) => { + return Err(Error::Plan { + field: Some(field.name.clone()), + reason: Reason::TargetOptions, + }) + } + }; // A null has an equality term and no other, so a sealed null // needs a type whose terms are equality's alone. if field.null == Some(NullMode::Sealed) @@ -857,6 +905,14 @@ impl Plan { if let Some(reason) = field.null_refusal() { return Err(refuse(reason)); } + // A JSON document is stored through a target that stores one; + // a target's options belong to a target. + if !field.is_target() && field.field_type == Some(ValueKind::Json) { + return Err(refuse(Reason::JsonNeedsTarget)); + } + if !field.is_target() && field.target_options.is_some() { + return Err(refuse(Reason::TargetOptions)); + } // An indexed field declares its type: every value's term derives // from the one declared kind, never from whatever tag each value // arrived with. A target field's kind is the type's own. @@ -1055,6 +1111,20 @@ pub fn plan(frame: &[u8]) -> Result { /// target field needs a column for its label, so it is refused under a /// context field ([`TargetError::ContextField`]). /// +/// `"target"` may instead be an object of the name and the options of the +/// index the type carries: a type that stores a JSON document +/// (`EncryptedJson`) takes the JSON index's options object, as the +/// `{"json": ...}` output spells it, and no other type takes any. +/// +/// ```text +/// "target": { "name": "EncryptedJson", +/// "options": { "sealed_root": , "ordered": [, ...] } } +/// ``` +/// +/// Such a field declares the `"json"` type (or leaves it to the target), +/// hands each document over as UTF-8 JSON text, and stores a null as SQL +/// `NULL` (`"null": "sql"`): a document has no length to pad. +/// /// `` is an index in its wire form, which is its key — `"eq"`, /// `"match"`, `"ore"` or `"ope"` — save for a match index with options other /// than the defaults, which is a one-entry object mapping `"match"` to them: @@ -1169,7 +1239,13 @@ pub fn plan_with(frame: &[u8], resolver: &(impl TargetResolver + ?Sized)) -> Res let context = super::context::non_empty(spec.context.0).map_err(|e| e.in_field(&name))?; let field = match spec.what { What::Outputs(outputs) => FieldPlan::new(name, context, outputs)?, - What::Target(target) => FieldPlan::with_target(name, context, target)?, + What::Target(target, options) => { + let field = FieldPlan::with_target(name, context, target)?; + match options { + Some(options) => field.with_target_options(options), + None => field, + } + } }; let field = match spec.field_type { Some(KindName(field_type)) => field.with_type(field_type)?, @@ -1291,7 +1367,7 @@ struct FieldSpec { enum What { Outputs(Vec), - Target(String), + Target(String, Option), } #[derive(Deserialize)] @@ -1301,7 +1377,7 @@ struct FieldSpecWire { #[serde(default, deserialize_with = "present")] outputs: Option>, #[serde(default, deserialize_with = "present")] - target: Option, + target: Option, #[serde(default, deserialize_with = "present", rename = "type")] field_type: Option, #[serde(default, deserialize_with = "present")] @@ -1314,7 +1390,10 @@ impl TryFrom for FieldSpec { fn try_from(wire: FieldSpecWire) -> Result { let what = match (wire.outputs, wire.target) { (Some(outputs), None) => What::Outputs(outputs.into_iter().map(Output::from).collect()), - (None, Some(target)) => What::Target(target), + (None, Some(TargetWire::Name(target))) => What::Target(target, None), + (None, Some(TargetWire::Configured(ConfiguredTarget { name, options }))) => { + What::Target(name, options.map(|options| TargetOptions::Json(options.0))) + } (Some(_), Some(_)) => return Err(Reason::OutputsWithTarget), (None, None) => return Err(Reason::MissingOutputs), }; @@ -1327,6 +1406,25 @@ impl TryFrom for FieldSpec { } } +/// A field's `"target"`: the target's name, which runs it under its +/// defaults, or an object of its name and its options, which are the +/// options of the index the target carries (the JSON index's options +/// object, as an output spells it). +#[derive(Deserialize)] +#[serde(untagged)] +enum TargetWire { + Name(String), + Configured(ConfiguredTarget), +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct ConfiguredTarget { + name: String, + #[serde(default)] + options: Option, +} + /// Absent is `None`. Present, a value must be of its kind: a null is /// refused, as any other value of the wrong kind is. fn present<'de, D: Deserializer<'de>, T: Deserialize<'de>>(d: D) -> Result, D::Error> { @@ -1560,7 +1658,7 @@ pub fn encrypt_with<'a, K: 'static>( let name = field.target().unwrap_or_default(); pendings.push( resolver - .encrypt(name, cipher, field.label(), value) + .encrypt(name, field.target_options(), cipher, field.label(), value) .map_err(|error| Error::Target(name_target(error, &field.name)))?, ); count += 1; @@ -1751,6 +1849,11 @@ fn name_target(error: TargetError, field: &str) -> TargetError { target, reason, }, + TargetError::Json { target, reason, .. } => TargetError::Json { + name: field.to_owned(), + target, + reason, + }, other => other, } } @@ -2285,7 +2388,7 @@ fn check_field(value: &ValueFrame, field: &FieldPlan) -> Result<(), Error> { reason, }; if let Some(declared) = field.field_type { - if value.kind() != Some(declared) { + if !declared.holds(value.kind()) { return Err(misfit(Reason::FieldType)); } } @@ -2465,7 +2568,7 @@ fn open_record( // opens as a plain null, as every frame carries one. let null = field.null.is_some() && value.is_null(); if let Some(kind) = field.kind.filter(|_| !null) { - if value.kind() != Some(kind) { + if !kind.holds(value.kind()) { return Err(crate::PlanError::FieldType { field: field.name.clone(), expected: kind.name(), @@ -2593,7 +2696,7 @@ fn record_row(mut row: Vec<(String, StackCipherText)>, plan: &Plan) -> Result( &self, name: &str, + _: Option<&crate::dynamic::TargetOptions>, keyset: &'a KeysetCipher<'_, K>, label: &Label, plaintext: ValueFrame, @@ -6431,6 +6547,137 @@ mod tests { .expect("seals")) } + /// A target that stores a JSON document takes the JSON index's + /// options, in a `{"name", "options"}` target; given none, it runs + /// under the index's defaults, resolved when the plan is built. + #[test] + fn a_json_target_takes_the_index_options_and_defaults_them() { + use crate::dynamic::TargetOptions; + use crate::json::{JsonOptions, JsonPath}; + let bare = plan_with( + obj(vec![( + "profile", + target_spec(label("profile"), "EncryptedJson"), + )]), + &FakeEql, + ) + .expect("a bare JSON target"); + let field = &bare.fields()[0]; + assert_eq!( + field.field_type(), + Some(ValueKind::Json), + "the target's kind" + ); + assert_eq!( + field.target_options(), + Some(&TargetOptions::Json(JsonOptions::default())) + ); + let configured = |options: Tree| { + plan_with( + obj(vec![( + "profile", + obj(vec![ + ("context", label("profile")), + ( + "target", + obj(vec![("name", s("EncryptedJson")), ("options", options)]), + ), + ]), + )]), + &FakeEql, + ) + }; + let plan = configured(obj(vec![ + ("sealed_root", Tree::Bool(false)), + ( + "ordered", + Tree::Array(vec![s("$.salary"), s("$.visits[*].at")]), + ), + ])) + .expect("configured"); + let Some(TargetOptions::Json(options)) = plan.fields()[0].target_options() else { + panic!("JSON options"); + }; + assert!(!options.sealed_root); + assert!(options.orders(&JsonPath::parse("$.salary").unwrap())); + assert!(options.orders(&JsonPath::parse("$.visits[3].at").unwrap())); + for (options, reason) in [ + ( + obj(vec![("sealed", Tree::Bool(true))]), + Reason::InvalidTarget, + ), + ( + obj(vec![("ordered", Tree::Array(vec![s("$.scores[2]")]))]), + Reason::InvalidTarget, + ), + ( + obj(vec![("ordered", Tree::Array(vec![s("not a path")]))]), + Reason::InvalidTarget, + ), + ] { + let error = refused(configured(options)); + assert_eq!(error.reason(), Some(reason), "{error}"); + assert_eq!(error.field(), Some("profile")); + } + } + + #[test] + fn options_on_a_target_that_takes_none_are_refused() { + let error = refused(plan_with( + obj(vec![( + "email", + obj(vec![ + ("context", label("email")), + ( + "target", + obj(vec![("name", s(TEXT_EQ)), ("options", obj(vec![]))]), + ), + ]), + )]), + &FakeEql, + )); + assert_eq!(error.reason(), Some(Reason::TargetOptions), "{error}"); + let error = refused(plan_with( + obj(vec![( + "email", + obj(vec![ + ("context", label("email")), + ( + "target", + obj(vec![("name", s(TEXT_EQ)), ("version", Tree::UInt32(1))]), + ), + ]), + )]), + &FakeEql, + )); + assert_eq!( + error.reason(), + Some(Reason::InvalidTarget), + "an unknown key: {error}" + ); + } + + /// A `json` field names a target that stores the document; a JSON + /// document has no length to pad, so it stores a null as SQL NULL. + #[test] + fn a_json_field_needs_a_document_target_and_sql_nulls() { + let error = refused(plan_with( + obj(vec![("profile", typed(label("profile"), &["c"], "json"))]), + &FakeEql, + )); + assert_eq!(error.reason(), Some(Reason::JsonNeedsTarget), "{error}"); + let mut spec = target_spec(label("profile"), "EncryptedJson"); + if let Tree::Object(entries) = &mut spec { + entries.push(("null".to_string(), s("sealed"))); + } + let error = refused(plan_with(obj(vec![("profile", spec.clone())]), &FakeEql)); + assert_eq!(error.reason(), Some(Reason::SealedNullIndex), "{error}"); + if let Tree::Object(entries) = &mut spec { + entries.last_mut().unwrap().1 = s("sql"); + } + assert!(plan_with(obj(vec![("profile", spec)]), &FakeEql).is_ok()); + } + #[tokio::test] async fn is_sealed_by_the_resolver_in_the_records_one_request() { let cipher = cipher().await; diff --git a/packages/stack-encrypt/src/dynamic/target.rs b/packages/stack-encrypt/src/dynamic/target.rs index e34902c2d..3593c2c3e 100644 --- a/packages/stack-encrypt/src/dynamic/target.rs +++ b/packages/stack-encrypt/src/dynamic/target.rs @@ -30,6 +30,7 @@ use std::fmt; use super::query::target_operators; use super::{OpSpec, ValueFrame, ValueKind}; +use crate::json::JsonOptions; use crate::sem::MatchOptions; use crate::target::IndexSpec; use crate::{KeysetCipher, Label, Pending, StackCipher}; @@ -209,6 +210,32 @@ impl TargetDescriptor { } } +/// The options a plan field gives the EQL type it names: the options of +/// the index the type carries. A target that stores a JSON document +/// (`EncryptedJson`) carries the JSON index and takes its options, the +/// sealed root and the ordered paths; no other target takes any today. +/// +/// The plan resolves them once, when it is built ([`Plan::new_with`]): +/// a JSON target given none runs under the index's defaults, so the +/// field's writes, queries and reads run under one definition. +/// +/// [`Plan::new_with`]: super::record::Plan::new_with +#[derive(Clone, Debug, PartialEq, Eq)] +#[non_exhaustive] +pub enum TargetOptions { + /// The JSON index's options. + Json(JsonOptions), +} + +impl TargetOptions { + /// The JSON index's options, if these are they. + pub fn json(&self) -> Option<&JsonOptions> { + match self { + TargetOptions::Json(options) => Some(options), + } + } +} + /// Why a target field could not be resolved or run. /// /// Every variant but [`Other`](Self::Other) is a statement about the plan, @@ -336,6 +363,22 @@ pub enum TargetError { /// passthrough value. found: Option, }, + /// The value is not a JSON document by the engine's rules + /// ([`crate::json::parse_text`]): not text or bytes, not JSON, an + /// object that names a member twice, or an integer literal outside the + /// 64-bit range. Decided before any key is touched. + #[error("{name}: {target} stores a JSON document, and the value is not one: {reason}")] + #[diagnostic(code(stack_encrypt::target_json))] + Json { + /// The field's name. + name: String, + /// The EQL type. + target: String, + /// What was wrong, by its code ([`super::JsonValueError::code`]): + /// `json_not_text`, `json_syntax`, `json_duplicate_key` or + /// `json_integer_range`. Never a byte of the value. + reason: &'static str, + }, /// The stored bytes are not a value of the type. #[error("{name}: the stored value is not a {target}: {reason}")] #[diagnostic(code(stack_encrypt::target_stored))] @@ -404,6 +447,10 @@ impl crate::ErrorPayload for TargetError { ("target", target.as_str().into()), ("reason", reason.as_str().into()), ]), + Self::Json { target, reason, .. } => payload([ + ("target", target.as_str().into()), + ("reason", (*reason).into()), + ]), }; // `name` is the target type for the first four, and the field for // the rest (the lowering fills it in): keep the two apart. @@ -419,6 +466,7 @@ impl crate::ErrorPayload for TargetError { | Self::Kind { name, .. } | Self::Column { name, .. } | Self::Plaintext { name, .. } + | Self::Json { name, .. } | Self::Stored { name, .. } => { if !name.is_empty() { let _ = fields.insert("field".to_owned(), name.as_str().into()); @@ -482,7 +530,9 @@ pub trait TargetResolver { } /// Run the named type's own encryption plan over one value under - /// `label`, resolving to the EQL value's JSON bytes. + /// `label`, resolving to the EQL value's JSON bytes. `options` are the + /// ones the plan resolved for the field ([`TargetOptions`]): the JSON + /// index's for a type that carries it, `None` for every other type. /// /// # Errors /// @@ -493,6 +543,7 @@ pub trait TargetResolver { fn encrypt<'a, K: 'static>( &self, name: &str, + options: Option<&TargetOptions>, keyset: &'a KeysetCipher<'_, K>, label: &Label, plaintext: ValueFrame, @@ -565,6 +616,7 @@ impl TargetResolver for NoTargets { fn encrypt<'a, K: 'static>( &self, name: &str, + _: Option<&TargetOptions>, _: &'a KeysetCipher<'_, K>, _: &Label, _: ValueFrame, @@ -645,6 +697,7 @@ mod tests { fn encrypt<'a, K: 'static>( &self, _: &str, + _: Option<&TargetOptions>, _: &'a KeysetCipher<'_, K>, _: &Label, _: ValueFrame, diff --git a/packages/stack-encrypt/src/json/mod.rs b/packages/stack-encrypt/src/json/mod.rs index a145b913d..fe6539814 100644 --- a/packages/stack-encrypt/src/json/mod.rs +++ b/packages/stack-encrypt/src/json/mod.rs @@ -122,12 +122,25 @@ mod derive; mod path; mod plain; mod query; +mod text; pub use derive::JsonTerms; pub use path::{JsonPath, JsonPathError, Segment}; pub use query::{ AtPathTerms, JsonLeafQuery, JsonProjection, JsonQuery, JsonSelector, JsonSubtreeKey, }; +pub use text::{parse_text, to_text, JsonTextError}; + +/// Wipe a JSON value's plaintext in place: every string (object keys too) +/// is overwritten, every number set to zero and every boolean to `false`, +/// and the containers emptied. For a binding that holds a document it +/// parsed ([`parse_text`]) or opened, once it is done with it: a +/// `serde_json::Value` has no `Zeroize` of its own. Numbers and booleans +/// are overwritten best-effort, as [`Zeroize`](zeroize::Zeroize) on the +/// engine's own JSON plaintext does. +pub fn wipe_value(value: &mut serde_json::Value) { + plain::wipe(value); +} pub(crate) use derive::{seal_pending, terms_pending}; diff --git a/packages/stack-encrypt/src/json/plain.rs b/packages/stack-encrypt/src/json/plain.rs index 6f7b96096..27411913c 100644 --- a/packages/stack-encrypt/src/json/plain.rs +++ b/packages/stack-encrypt/src/json/plain.rs @@ -69,7 +69,7 @@ impl Zeroize for PlainJson { /// Overwrite every scalar and key in `value` in place, then empty its /// containers. -fn wipe(value: &mut Value) { +pub(crate) fn wipe(value: &mut Value) { match value { Value::String(text) => text.zeroize(), Value::Number(number) => *number = Number::from(0u8), diff --git a/packages/stack-encrypt/src/json/text.rs b/packages/stack-encrypt/src/json/text.rs new file mode 100644 index 000000000..95fe863a9 --- /dev/null +++ b/packages/stack-encrypt/src/json/text.rs @@ -0,0 +1,348 @@ +//! JSON text at the engine's boundary: how a language SDK hands the JSON +//! index a document, and how it gets one back. +//! +//! A binding has no `serde_json::Value` to pass. It sends the document as +//! UTF-8 JSON text, and the engine parses it here, once, by one set of +//! rules, so every language seals the same document for the same text: +//! +//! - **Numbers** are signed or unsigned 64-bit integers, or binary64 +//! floats. An integer literal (no fraction, no exponent) outside +//! `i64::MIN ..= u64::MAX` is refused ([`JsonTextError::IntegerRange`]) +//! rather than read as the nearest `f64`, which would seal a different +//! number than the text names. A literal with a fraction or an exponent +//! is the nearest binary64 (serde_json's `float_roundtrip` parser), so +//! `1e400` is refused as not JSON this engine reads, and `0.1` is the +//! double nearest to it. +//! - **An object names each member once.** A repeated key is refused +//! ([`JsonTextError::DuplicateKey`]) rather than resolved, since two JSON +//! parsers keep different ones (Go's `encoding/json` the last, others the +//! first) and the sealed document would depend on which ran. +//! - **Text is valid Unicode**: invalid UTF-8 and an unpaired surrogate +//! escape (`"\ud800"`) are refused as not JSON. +//! - **Nesting** is limited by the parser (128 levels) and by the index +//! ([`TermError::JsonTooDeep`](crate::sem::TermError::JsonTooDeep)). +//! +//! What comes back out ([`to_text`]) is JSON text serde_json writes: +//! numbers in their shortest round-tripping form, members in the order the +//! document holds them. Key order, whitespace and the spelling of a number +//! (`1.0`, `1e0`) are not promised to survive: the document is sealed as +//! values, not as text. +//! +//! Every error names a class and, for a syntax error, a line and column, +//! and never a byte of the text. + +use std::fmt; + +use serde::de::{DeserializeSeed, Deserializer, Error as _, MapAccess, SeqAccess, Visitor}; +use serde_json::{Map, Number, Value}; + +/// Why JSON text was refused at the boundary. Each says what kind of +/// mistake it was and where, never what the text held. +#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)] +#[non_exhaustive] +pub enum JsonTextError { + /// Not JSON text: a syntax error, text that is not UTF-8, an unpaired + /// surrogate, a float out of range, nesting past the parser's limit, or + /// anything after the value. + #[error("not JSON text (line {line}, column {column})")] + Syntax { + /// The line of the first byte refused, from 1. + line: usize, + /// Its column, from 1. + column: usize, + }, + /// An object names one member twice. + #[error("an object names a member twice (line {line}, column {column})")] + DuplicateKey { + /// The line of the second name, from 1. + line: usize, + /// Its column, from 1. + column: usize, + }, + /// An integer literal outside the signed and unsigned 64-bit range. + #[error("an integer literal is outside the 64-bit range")] + IntegerRange, +} + +impl JsonTextError { + /// The error's machine-readable code: `json_syntax`, + /// `json_duplicate_key` or `json_integer_range`. + pub fn code(&self) -> &'static str { + match self { + JsonTextError::Syntax { .. } => "json_syntax", + JsonTextError::DuplicateKey { .. } => "json_duplicate_key", + JsonTextError::IntegerRange => "json_integer_range", + } + } +} + +/// The marker a duplicate key is raised with inside the parse, told apart +/// from serde_json's own errors by its text, which nothing else writes. +const DUPLICATE: &str = "\u{0}stack-encrypt: duplicate member"; + +/// Parse a document from UTF-8 JSON text by the rules in the module docs. +/// +/// # Errors +/// +/// [`JsonTextError`]: the text is not JSON, an object repeats a member, +/// or an integer literal is out of range. +/// +/// ``` +/// use stack_encrypt::json::{parse_text, JsonTextError}; +/// +/// let value = parse_text(br#"{"name": "Alice", "age": 34}"#)?; +/// assert_eq!(value["age"], 34); +/// assert!(matches!(parse_text(br#"{"a": 1, "a": 2}"#), Err(JsonTextError::DuplicateKey { .. }))); +/// assert_eq!(parse_text(b"18446744073709551616"), Err(JsonTextError::IntegerRange)); +/// # Ok::<(), JsonTextError>(()) +/// ``` +pub fn parse_text(text: &[u8]) -> Result { + check_integers(text)?; + let mut de = serde_json::Deserializer::from_slice(text); + let value = Strict.deserialize(&mut de).map_err(classify)?; + de.end().map_err(classify)?; + Ok(value) +} + +/// The document as JSON text: what opening returns to a binding. +/// +/// # Errors +/// +/// None in practice: every `Value` serializes. The `Result` is serde_json's. +pub fn to_text(value: &Value) -> Result, serde_json::Error> { + serde_json::to_vec(value) +} + +fn classify(error: serde_json::Error) -> JsonTextError { + let (line, column) = (error.line(), error.column()); + if error.to_string().starts_with(DUPLICATE) { + JsonTextError::DuplicateKey { line, column } + } else { + JsonTextError::Syntax { line, column } + } +} + +/// Refuse an integer literal serde_json would read as the nearest `f64`: +/// one outside `i64::MIN ..= u64::MAX`. A lexical scan, outside strings, +/// of each number token with no fraction or exponent. Syntax errors are +/// left to the parser. +fn check_integers(text: &[u8]) -> Result<(), JsonTextError> { + let mut i = 0; + while i < text.len() { + match text[i] { + b'"' => { + i += 1; + while i < text.len() && text[i] != b'"' { + // An escape takes the next byte with it, so `\"` does + // not end the string. + i += if text[i] == b'\\' { 2 } else { 1 }; + } + i += 1; + } + b'-' | b'0'..=b'9' => { + let start = i; + i += 1; + while i < text.len() + && matches!(text[i], b'0'..=b'9' | b'.' | b'e' | b'E' | b'+' | b'-') + { + i += 1; + } + let token = &text[start..i]; + if token.iter().any(|b| matches!(b, b'.' | b'e' | b'E')) { + continue; + } + // ASCII by construction; a malformed token is the parser's. + let Ok(token) = std::str::from_utf8(token) else { + continue; + }; + let digits = token.strip_prefix('-').unwrap_or(token); + if digits.is_empty() || !digits.bytes().all(|b| b.is_ascii_digit()) { + continue; + } + let fits = if token.starts_with('-') { + token.parse::().is_ok() + } else { + token.parse::().is_ok() + }; + if !fits { + return Err(JsonTextError::IntegerRange); + } + } + _ => i += 1, + } + } + Ok(()) +} + +/// A `Value`, refusing an object that names a member twice. +struct Strict; + +impl<'de> DeserializeSeed<'de> for Strict { + type Value = Value; + fn deserialize>(self, d: D) -> Result { + d.deserialize_any(StrictVisitor) + } +} + +struct StrictVisitor; + +impl<'de> Visitor<'de> for StrictVisitor { + type Value = Value; + + fn expecting(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("a JSON value") + } + + fn visit_bool(self, v: bool) -> Result { + Ok(Value::Bool(v)) + } + + fn visit_i64(self, v: i64) -> Result { + Ok(Value::Number(v.into())) + } + + fn visit_u64(self, v: u64) -> Result { + Ok(Value::Number(v.into())) + } + + fn visit_f64(self, v: f64) -> Result { + Number::from_f64(v) + .map(Value::Number) + .ok_or_else(|| E::custom("a number that is not finite")) + } + + fn visit_str(self, v: &str) -> Result { + Ok(Value::String(v.to_owned())) + } + + fn visit_string(self, v: String) -> Result { + Ok(Value::String(v)) + } + + fn visit_unit(self) -> Result { + Ok(Value::Null) + } + + fn visit_none(self) -> Result { + Ok(Value::Null) + } + + fn visit_some>(self, d: D) -> Result { + Strict.deserialize(d) + } + + fn visit_seq>(self, mut seq: A) -> Result { + let mut items = Vec::new(); + while let Some(item) = seq.next_element_seed(Strict)? { + items.push(item); + } + Ok(Value::Array(items)) + } + + fn visit_map>(self, mut map: A) -> Result { + let mut members = Map::new(); + while let Some(key) = map.next_key::()? { + if members.contains_key(&key) { + return Err(A::Error::custom(DUPLICATE)); + } + let value = map.next_value_seed(Strict)?; + let _ = members.insert(key, value); + } + Ok(Value::Object(members)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn reads_every_kind_of_value() { + let value = parse_text( + br#"{"s": "x", "n": -3, "u": 18446744073709551615, "f": 1.5, "b": true, "z": null, "a": [1, {}]}"#, + ) + .unwrap(); + assert_eq!( + value, + json!({"s": "x", "n": -3, "u": 18446744073709551615u64, "f": 1.5, "b": true, "z": null, "a": [1, {}]}) + ); + assert_eq!(parse_text(b" null ").unwrap(), Value::Null); + assert_eq!(parse_text(b"\"text\"").unwrap(), json!("text")); + } + + #[test] + fn the_64_bit_bounds_are_integers_and_one_past_is_refused() { + assert_eq!( + parse_text(b"-9223372036854775808").unwrap(), + json!(i64::MIN) + ); + assert_eq!( + parse_text(b"18446744073709551615").unwrap(), + json!(u64::MAX) + ); + for text in [ + &b"-9223372036854775809"[..], + b"18446744073709551616", + br#"{"a": [1, 99999999999999999999]}"#, + ] { + assert_eq!(parse_text(text), Err(JsonTextError::IntegerRange)); + } + // With a fraction or an exponent it is a float, by the documented + // binary64 rule. + assert_eq!(parse_text(b"1e20").unwrap(), json!(1e20)); + assert_eq!( + parse_text(b"18446744073709551616.0").unwrap(), + json!(1.8446744073709552e19) + ); + } + + #[test] + fn a_number_inside_a_string_is_text() { + assert_eq!( + parse_text(br#"{"k": "99999999999999999999", "e": "a\"99999999999999999999"}"#) + .unwrap()["k"], + json!("99999999999999999999") + ); + } + + #[test] + fn a_repeated_member_is_refused_with_its_place() { + assert_eq!( + parse_text(b"{\"a\": 1,\n \"a\": 2}"), + Err(JsonTextError::DuplicateKey { line: 2, column: 4 }) + ); + assert!(matches!( + parse_text(br#"[{"x": {"y": 1, "y": 1}}]"#), + Err(JsonTextError::DuplicateKey { .. }) + )); + // The same name in two objects is two members. + assert!(parse_text(br#"[{"a": 1}, {"a": 2}]"#).is_ok()); + } + + #[test] + fn text_that_is_not_json_is_refused_without_quoting_it() { + for text in [ + &b""[..], + b"{", + b"{\"secret-4242\"}", + b"[1] [2]", + b"\"\\ud800\"", + b"\"\xff\"", + b"1e400", + b"NaN", + ] { + let error = parse_text(text).unwrap_err(); + assert!(matches!(error, JsonTextError::Syntax { .. }), "{error:?}"); + assert!(!error.to_string().contains("4242")); + assert_eq!(error.code(), "json_syntax"); + } + } + + #[test] + fn text_round_trips_by_value() { + let value = parse_text(br#"{"b": 1.0, "a": [0.1, -0.0]}"#).unwrap(); + let text = to_text(&value).unwrap(); + assert_eq!(parse_text(&text).unwrap(), value); + } +} From 953bf0d6046039a5a984d420f9244174108c78dc Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 00:30:03 +1100 Subject: [PATCH 02/21] feat(go): name EncryptedJson in a plan and generate its EQL types encrypt/eql gains the generated EncryptedJSON, EncryptedJSONQuery, EncryptedJSONLeafQuery and EncryptedJSONEntry. The Go-name rule, here and in eql-codegen, spells the word Json as JSON wherever it is a word, not only as a prefix (EncryptedJson is EncryptedJSON). internal/record carries the `json` kind and a target's options (Field.TargetOptions), written as the `{"name", "options"}` target the engine reads. A hand-built plan can store and open a document through the guest with the EQL types: json_tree_test.go round-trips scalar roots, arrays, null and a u64, and shows a refused document is ErrEncoding with the engine's code and none of the text. stashgen support follows. Refs #1347, #1348, #1349 Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- languages/golang/CHANGELOG.md | 8 ++ languages/golang/cmd/stashgen/main_test.go | 5 +- languages/golang/encrypt/eql/eql_gen.go | 94 +++++++++++++++ languages/golang/encrypt/eql/eql_test.go | 1 + languages/golang/encrypt/json_tree_test.go | 107 ++++++++++++++++++ languages/golang/internal/record/record.go | 59 +++++++++- .../golang/internal/record/record_test.go | 38 +++++++ languages/golang/stashgen/read.go | 27 ++++- 8 files changed, 327 insertions(+), 12 deletions(-) create mode 100644 languages/golang/encrypt/json_tree_test.go diff --git a/languages/golang/CHANGELOG.md b/languages/golang/CHANGELOG.md index ea21c5ed1..c390530a7 100644 --- a/languages/golang/CHANGELOG.md +++ b/languages/golang/CHANGELOG.md @@ -12,6 +12,14 @@ listed under **Breaking**. Releases are tagged `languages/golang/vX.Y.Z`. ### Added +- **The EQL types for an encrypted JSON document** (#1348): + `eql.EncryptedJSON` (`public.eql_v4_json_tree`), its containment needle + `eql.EncryptedJSONQuery`, its ordering operand + `eql.EncryptedJSONLeafQuery` and its entry `eql.EncryptedJSONEntry`, + generated from the EQL catalog. A plan built by hand can store one now + (`record.JSON`, `record.Field.TargetOptions`); `stashgen` support + follows. + - **Decimals, dates and timestamps take an order index (#1236).** An `encrypt.Decimal`, `encrypt.Date` or `time.Time` field takes `ore` and `ope`, and the generated code gives it `Lt`, `Lte`, `Gt`, `Gte` and diff --git a/languages/golang/cmd/stashgen/main_test.go b/languages/golang/cmd/stashgen/main_test.go index a643ab0f3..298a67a2b 100644 --- a/languages/golang/cmd/stashgen/main_test.go +++ b/languages/golang/cmd/stashgen/main_test.go @@ -259,8 +259,9 @@ func TestEQLGoNameAgreesWithTheGeneratedPackage(t *testing.T) { if renamed == 0 { t.Fatal("the rule renamed nothing: the Json family is spelled JSON") } - if stashgen.EQLGoName("Json") != "JSON" || stashgen.EQLGoName("TextEq") != "TextEq" || stashgen.EQLGoName("SteVecQuery") != "SteVecQuery" { - t.Fatal("the one rule: a Json prefix is JSON, every other name is itself") + if stashgen.EQLGoName("Json") != "JSON" || stashgen.EQLGoName("TextEq") != "TextEq" || stashgen.EQLGoName("SteVecQuery") != "SteVecQuery" || + stashgen.EQLGoName("EncryptedJsonLeafQuery") != "EncryptedJSONLeafQuery" || stashgen.EQLGoName("Jsonify") != "Jsonify" { + t.Fatal("the one rule: the word Json is JSON, every other name is itself") } } diff --git a/languages/golang/encrypt/eql/eql_gen.go b/languages/golang/encrypt/eql/eql_gen.go index 17f9f8a6a..b8407bb23 100644 --- a/languages/golang/encrypt/eql/eql_gen.go +++ b/languages/golang/encrypt/eql/eql_gen.go @@ -60,6 +60,7 @@ var Types = []Type{ {Name: "DoubleOrdOpe", GoName: "DoubleOrdOpe", Family: "double", Suffix: "OrdOpe", Plaintext: "float64", SQLDomain: "public.eql_v4_double_ord_ope", Indexes: []string{"ope"}, Query: "DoubleOrdOpeQuery", QuerySQLDomain: "eql_v4.query_double_ord_ope", Producible: true, Reason: ""}, {Name: "SteVecDocument", GoName: "SteVecDocument", Family: "json", Suffix: "Search", Plaintext: "", SQLDomain: "public.eql_v4_json_search", Indexes: []string{"json"}, Query: "SteVecQuery", QuerySQLDomain: "eql_v4.query_json", Producible: false, Reason: "the JSON index is a new operation in the engine (#1060)"}, {Name: "Json", GoName: "JSON", Family: "json", Suffix: "", Plaintext: "", SQLDomain: "public.eql_v4_json", Indexes: nil, Query: "", QuerySQLDomain: "", Producible: false, Reason: "a JSON value seals as a tree of leaves and EQL's `c` holds one; how a document becomes one sealed leaf is not decided"}, + {Name: "EncryptedJson", GoName: "EncryptedJSON", Family: "json_tree", Suffix: "", Plaintext: "json", SQLDomain: "public.eql_v4_json_tree", Indexes: []string{"json"}, Query: "EncryptedJsonQuery", QuerySQLDomain: "eql_v4.query_json_tree", Producible: true, Reason: ""}, } // Integer is the EQL type stored in public.eql_v4_integer: the integer family with no index. @@ -2206,6 +2207,99 @@ func (v *JSON) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } +// EncryptedJSON is the EQL type stored in public.eql_v4_json_tree: the json_tree family with the json index. +// The engine produces it from a json. +type EncryptedJSON []byte + +// Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. +func (v EncryptedJSON) Value() (driver.Value, error) { + return value(v) +} + +// Scan reads a public.eql_v4_json_tree column. +func (v *EncryptedJSON) Scan(src any) error { + return scan((*[]byte)(v), src, "EncryptedJSON") +} + +// MarshalJSON is the EQL value as the JSON it is, or null when empty. +func (v EncryptedJSON) MarshalJSON() ([]byte, error) { + return marshal(v, "EncryptedJSON") +} + +// UnmarshalJSON reads the EQL value from JSON, as it is. +func (v *EncryptedJSON) UnmarshalJSON(b []byte) error { + return unmarshal((*[]byte)(v), b) +} + +// EncryptedJSONQuery is the query value for EncryptedJSON: the operand eql_v4.query_json_tree takes. +type EncryptedJSONQuery []byte + +// Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. +func (v EncryptedJSONQuery) Value() (driver.Value, error) { + return value(v) +} + +// Scan reads a eql_v4.query_json_tree column. +func (v *EncryptedJSONQuery) Scan(src any) error { + return scan((*[]byte)(v), src, "EncryptedJSONQuery") +} + +// MarshalJSON is the EQL value as the JSON it is, or null when empty. +func (v EncryptedJSONQuery) MarshalJSON() ([]byte, error) { + return marshal(v, "EncryptedJSONQuery") +} + +// UnmarshalJSON reads the EQL value from JSON, as it is. +func (v *EncryptedJSONQuery) UnmarshalJSON(b []byte) error { + return unmarshal((*[]byte)(v), b) +} + +// EncryptedJSONLeafQuery is the json_tree family's the ordering operand at a path, compared with an entry `->` extracts: the value eql_v4.query_json_tree_leaf holds. +type EncryptedJSONLeafQuery []byte + +// Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. +func (v EncryptedJSONLeafQuery) Value() (driver.Value, error) { + return value(v) +} + +// Scan reads a eql_v4.query_json_tree_leaf column. +func (v *EncryptedJSONLeafQuery) Scan(src any) error { + return scan((*[]byte)(v), src, "EncryptedJSONLeafQuery") +} + +// MarshalJSON is the EQL value as the JSON it is, or null when empty. +func (v EncryptedJSONLeafQuery) MarshalJSON() ([]byte, error) { + return marshal(v, "EncryptedJSONLeafQuery") +} + +// UnmarshalJSON reads the EQL value from JSON, as it is. +func (v *EncryptedJSONLeafQuery) UnmarshalJSON(b []byte) error { + return unmarshal((*[]byte)(v), b) +} + +// EncryptedJSONEntry is the json_tree family's one node of a document, as `->` and the walks return it: the value public.eql_v4_json_tree_entry holds. +type EncryptedJSONEntry []byte + +// Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. +func (v EncryptedJSONEntry) Value() (driver.Value, error) { + return value(v) +} + +// Scan reads a public.eql_v4_json_tree_entry column. +func (v *EncryptedJSONEntry) Scan(src any) error { + return scan((*[]byte)(v), src, "EncryptedJSONEntry") +} + +// MarshalJSON is the EQL value as the JSON it is, or null when empty. +func (v EncryptedJSONEntry) MarshalJSON() ([]byte, error) { + return marshal(v, "EncryptedJSONEntry") +} + +// UnmarshalJSON reads the EQL value from JSON, as it is. +func (v *EncryptedJSONEntry) UnmarshalJSON(b []byte) error { + return unmarshal((*[]byte)(v), b) +} + // IntegerRangeOreQuery is a range query value over the ore_block index of the integer family: both bounds of one BETWEEN, // included, as the one operand eql_v4.query_integer_range_ore takes in eql_v4.between. It ranges over IntegerOrdOre. type IntegerRangeOreQuery []byte diff --git a/languages/golang/encrypt/eql/eql_test.go b/languages/golang/encrypt/eql/eql_test.go index fc8264d2e..0b3df6878 100644 --- a/languages/golang/encrypt/eql/eql_test.go +++ b/languages/golang/encrypt/eql/eql_test.go @@ -30,6 +30,7 @@ func TestTypesTableNamesTheProducibleTypes(t *testing.T) { "Boolean", "Real", "RealEq", "RealOrdOre", "RealOrd", "RealOrdOpe", "Double", "DoubleEq", "DoubleOrdOre", "DoubleOrd", "DoubleOrdOpe", + "EncryptedJson", } if len(producible) != len(want) { t.Fatalf("producible = %v, want %v", producible, want) diff --git a/languages/golang/encrypt/json_tree_test.go b/languages/golang/encrypt/json_tree_test.go new file mode 100644 index 000000000..b0a3db4c2 --- /dev/null +++ b/languages/golang/encrypt/json_tree_test.go @@ -0,0 +1,107 @@ +package encrypt_test + +import ( + "context" + "encoding/json" + "errors" + "reflect" + "strings" + "testing" + + "github.com/cipherstash/stack/languages/golang/encrypt" + "github.com/cipherstash/stack/languages/golang/encrypt/eql" + "github.com/cipherstash/stack/languages/golang/internal/record" +) + +// An encrypted JSON document (the EQL type EncryptedJson) through the guest +// that holds the EQL types: a field of kind json names the target with the +// JSON index's options, crosses as its JSON text, and opens back to it. + +func jsonPlan() *record.Plan { + return &record.Plan{Context: []string{"users"}, Fields: []record.Field{{ + Name: "profile", Kind: record.JSON, Target: "EncryptedJson", Null: record.SQLNull, + TargetOptions: &record.TargetOptions{Ordered: []string{"$.age"}}, + }}} +} + +var jsonDocuments = []string{ + `{"name":"Alice","age":34,"visits":[{"at":3},{"at":1}]}`, + `[1,"two",{"three":3}]`, + `"a scalar root"`, + `null`, + `18446744073709551615`, + `{}`, +} + +func TestJSONDocumentsRoundTripThroughTheGuest(t *testing.T) { + c := deterministicEQLClient(t) + ctx := context.Background() + rows := make([]record.Source, len(jsonDocuments)) + for i, doc := range jsonDocuments { + rows[i] = record.Source{"profile": []byte(doc)} + } + sealed, err := c.DefaultKeyset().Seal(ctx, jsonPlan(), rows) + if err != nil { + t.Fatal(err) + } + for i, s := range sealed { + var stored struct { + V int `json:"v"` + I map[string]string `json:"i"` + H string `json:"h"` + SV []json.RawMessage `json:"sv"` + } + if err := json.Unmarshal(s["profile"].EQL, &stored); err != nil { + t.Fatalf("row %d: %v", i, err) + } + if stored.V != 4 || stored.I["t"] != "users" || stored.I["c"] != "profile" || stored.H == "" || len(stored.SV) == 0 { + t.Fatalf("row %d: stored %+v", i, stored) + } + // The value scans and binds as the generated EQL type. + var col eql.EncryptedJSON + if err := col.Scan(s["profile"].EQL); err != nil { + t.Fatalf("row %d: Scan: %v", i, err) + } + } + opened, err := c.Open(ctx, jsonPlan(), sealed) + if err != nil { + t.Fatal(err) + } + for i, row := range opened { + text, ok := row["profile"].([]byte) + if !ok { + t.Fatalf("row %d opened as %T, not bytes", i, row["profile"]) + } + var got, want any + if err := json.Unmarshal(text, &got); err != nil { + t.Fatalf("row %d: %v", i, err) + } + _ = json.Unmarshal([]byte(jsonDocuments[i]), &want) + if !reflect.DeepEqual(got, want) { + t.Fatalf("row %d: opened %s, want %s", i, text, jsonDocuments[i]) + } + } +} + +// Text that is not a document is refused before anything is sealed, with +// the engine's code and without the text. +func TestJSONTextThatIsNotADocumentIsErrEncoding(t *testing.T) { + c := deterministicEQLClient(t) + for text, code := range map[string]string{ + `{"secret":1,"secret":2}`: "json_duplicate_key", + `99999999999999999999`: "json_integer_range", + `{"secret":`: "json_syntax", + } { + _, err := c.DefaultKeyset().Seal(context.Background(), jsonPlan(), []record.Source{{"profile": []byte(text)}}) + if !errors.Is(err, encrypt.ErrEncoding) { + t.Fatalf("%s: %v, want ErrEncoding", text, err) + } + var d *encrypt.Diagnostic + if !errors.As(err, &d) || d.Code != "stack_encrypt::target_json" || d.Reason() != code || d.Field() != "profile" { + t.Fatalf("%s: diagnostic %+v", text, d) + } + if strings.Contains(err.Error(), "secret") || strings.Contains(err.Error(), "9999") { + t.Fatalf("%s: the error holds the text: %v", text, err) + } + } +} diff --git a/languages/golang/internal/record/record.go b/languages/golang/internal/record/record.go index 9832f10dc..7f7b4b2e1 100644 --- a/languages/golang/internal/record/record.go +++ b/languages/golang/internal/record/record.go @@ -9,8 +9,8 @@ // dynamic::record: the output keys "c", "eq", "match", "ore", "ope", // "ore_block" and "passthrough", and the type names "bool", "int8", "int16", "int32", // "int64", "int128", "uint8", "uint16", "uint32", "uint64", "uint128", -// "float32", "float64", "decimal", "date", "timestamp", "string" and -// "bytes". A field's context is +// "float32", "float64", "decimal", "date", "timestamp", "string", +// "bytes" and "json". A field's context is // its label — the plan's context segments and the field's identity — nested // to the left under each part the caller extends it by. // @@ -62,6 +62,11 @@ const ( Timestamp Kind = "timestamp" String Kind = "string" Bytes Kind = "bytes" + // JSON is a JSON document, which crosses the binding as its UTF-8 JSON + // text in a bytes value and opens back to its text. The engine parses + // the text by one set of rules (numbers, repeated members, Unicode); + // a field of this kind names a target that stores a document. + JSON Kind = "json" ) // Output is one output of a sealed field, by its wire key. @@ -110,6 +115,41 @@ type Field struct { // field every row has, SealedNull or SQLNull for one declared over a // pointer. Null Null + // TargetOptions are the options the field gives its Target: those of + // the index the target carries. A target that stores a JSON document + // (EncryptedJson) takes the JSON index's; nil runs a target under its + // defaults, and no other target takes any. + TargetOptions *TargetOptions +} + +// TargetOptions are a target's options, as a plan gives them: the JSON +// index's, the only index that takes any. A field left nil takes the +// index's default. The engine checks them when the plan is built. +type TargetOptions struct { + // SealedRoot says whether the document's root is sealed: its entry + // holds the whole document, and no token enumerates its top-level + // members. nil is the default, sealed. + SealedRoot *bool + // Ordered are the JSONPath paths ($.age, $.visits[*].at) whose number + // and string leaves carry an ordering term, so they answer the + // orderings. None by default. + Ordered []string +} + +// wire is the options object as the plan spells it: only the options set. +func (o *TargetOptions) wire() vcvalue.Object { + out := vcvalue.Object{} + if o.SealedRoot != nil { + out = append(out, vcvalue.Field{Key: "sealed_root", Value: *o.SealedRoot}) + } + if o.Ordered != nil { + paths := make([]any, len(o.Ordered)) + for i, p := range o.Ordered { + paths[i] = p + } + out = append(out, vcvalue.Field{Key: "ordered", Value: paths}) + } + return out } // Null is how a field stores a value that is not there: the plan's "null". @@ -289,6 +329,9 @@ func (p *Plan) Validate() error { default: return fmt.Errorf("record: field %q: unknown null %q", f.Name, f.Null) } + if f.TargetOptions != nil && !f.IsTarget() { + return fmt.Errorf("record: field %q has target options and no target", f.Name) + } if f.IsTarget() { if len(f.Outputs) != 0 { return fmt.Errorf("record: field %q names the EQL type %s and outputs; a field has one or the other", f.Name, f.Target) @@ -333,7 +376,7 @@ func (p *Plan) Validate() error { func (k Kind) Known() bool { switch k { case Untyped, Bool, Int8, Int16, Int32, Int64, Int128, Uint8, Uint16, Uint32, Uint64, Uint128, - Float32, Float64, Decimal, Date, Timestamp, String, Bytes: + Float32, Float64, Decimal, Date, Timestamp, String, Bytes, JSON: return true } return false @@ -374,9 +417,15 @@ func (p *Plan) Wire() vcvalue.Object { } for _, f := range p.Fields { spec := vcvalue.Object{{Key: "context", Value: p.FieldContext(f)}} - if f.IsTarget() { + switch { + case f.IsTarget() && f.TargetOptions != nil: + spec = append(spec, vcvalue.Field{Key: "target", Value: vcvalue.Object{ + {Key: "name", Value: f.Target}, + {Key: "options", Value: f.TargetOptions.wire()}, + }}) + case f.IsTarget(): spec = append(spec, vcvalue.Field{Key: "target", Value: f.Target}) - } else { + default: outputs := make([]any, len(f.Outputs)) for i, o := range f.Outputs { outputs[i] = string(o) diff --git a/languages/golang/internal/record/record_test.go b/languages/golang/internal/record/record_test.go index 0feadcdc5..8c7c55b19 100644 --- a/languages/golang/internal/record/record_test.go +++ b/languages/golang/internal/record/record_test.go @@ -178,3 +178,41 @@ func TestParseContext(t *testing.T) { t.Fatalf("CheckSegment: %v", err) } } + +// A target with options is the object {"name", "options"}, with only the +// options set; a target with none is its name. +func TestWireGivesATargetItsOptions(t *testing.T) { + sealed := false + p := &Plan{Context: []string{"users"}, Fields: []Field{ + {Name: "profile", Kind: JSON, Target: "EncryptedJson", Null: SQLNull, TargetOptions: &TargetOptions{SealedRoot: &sealed, Ordered: []string{"$.age"}}}, + {Name: "prefs", Kind: JSON, Target: "EncryptedJson", Null: SQLNull, TargetOptions: &TargetOptions{}}, + {Name: "notes", Kind: JSON, Target: "EncryptedJson"}, + }} + if err := p.Validate(); err != nil { + t.Fatal(err) + } + wire := p.Wire() + target := func(i int) any { return wire[i].Value.(vcvalue.Object)[1].Value } + want := vcvalue.Object{{Key: "name", Value: "EncryptedJson"}, {Key: "options", Value: vcvalue.Object{ + {Key: "sealed_root", Value: false}, {Key: "ordered", Value: []any{"$.age"}}, + }}} + if !reflect.DeepEqual(target(0), want) { + t.Fatalf("target = %#v", target(0)) + } + if !reflect.DeepEqual(target(1), vcvalue.Object{{Key: "name", Value: "EncryptedJson"}, {Key: "options", Value: vcvalue.Object{}}}) { + t.Fatalf("empty options = %#v", target(1)) + } + if target(2) != "EncryptedJson" { + t.Fatalf("no options = %#v", target(2)) + } + if typ := wire[0].Value.(vcvalue.Object)[2]; typ.Key != "type" || typ.Value != "json" { + t.Fatalf("type = %#v", typ) + } + bad := &Plan{Context: []string{"users"}, Fields: []Field{{Name: "a", Kind: String, Outputs: []Output{Ciphertext}, TargetOptions: &TargetOptions{}}}} + if err := bad.Validate(); err == nil || !strings.Contains(err.Error(), "no target") { + t.Fatalf("options with no target: %v", err) + } + if !JSON.Known() { + t.Fatal("json is a kind") + } +} diff --git a/languages/golang/stashgen/read.go b/languages/golang/stashgen/read.go index 43e3aa2ce..663b36191 100644 --- a/languages/golang/stashgen/read.go +++ b/languages/golang/stashgen/read.go @@ -4,6 +4,7 @@ import ( "fmt" "go/types" "strings" + "unicode" "github.com/cipherstash/stack/languages/golang/internal/record" "golang.org/x/tools/go/packages" @@ -148,17 +149,33 @@ type modelField struct { } // EQLGoName is the Go type name in encrypt/eql of an EQL type the engine -// names: the same name in every language, save the JSON family, whose Go -// names start with JSON (Json is JSON), as Go spells initialisms. The +// names: the same name in every language, save that the word Json is JSON, +// as Go spells initialisms (Json is JSON, EncryptedJsonQuery is +// EncryptedJSONQuery; Json is a word where the name ends or the next +// letter is upper case). The // generated encrypt/eql package follows the same rule (eql-codegen's go_eql // renderer), and its Types table records both names, which is what holds // the two in agreement (a test in cmd/stashgen reads it; this package's own // tests cannot link encrypt/eql without changing the engine they test). func EQLGoName(name string) string { - if strings.HasPrefix(name, "Json") { - return "JSON" + strings.TrimPrefix(name, "Json") + var out strings.Builder + rest := name + for { + at := strings.Index(rest, "Json") + if at < 0 { + break + } + after := rest[at+4:] + out.WriteString(rest[:at]) + if after == "" || unicode.IsUpper(rune(after[0])) { + out.WriteString("JSON") + } else { + out.WriteString("Json") + } + rest = after } - return name + out.WriteString(rest) + return out.String() } // reader builds a genFile from a loaded package. From e2077b568a5586cb5941d58f8e1247b878f96c18 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 01:55:22 +1100 Subject: [PATCH 03/21] fix(stack-encrypt,go): build the check_record fuzz harness and skip json in the per-type PostgreSQL test The fuzz harness's TargetResolver still had encrypt's old signature, so fuzz regression (check-record) did not compile. The per-type PostgreSQL test failed on EncryptedJson, the first producible type with no plaintexts of its own: a document has operators of its own and is run in PostgreSQL with the generated fields. Refs #1347 Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- languages/golang/encrypt/eql_postgres_test.go | 6 ++++++ packages/stack-encrypt/fuzz/fuzz_targets/check_record.rs | 3 ++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/languages/golang/encrypt/eql_postgres_test.go b/languages/golang/encrypt/eql_postgres_test.go index 9772237cc..9f9d4d677 100644 --- a/languages/golang/encrypt/eql_postgres_test.go +++ b/languages/golang/encrypt/eql_postgres_test.go @@ -148,6 +148,12 @@ func TestEQLTypesRoundTripAndQueryInPostgres(t *testing.T) { ds[i] = d } runEQLType(h, gensupport.Decimal, ds, nil) + case "json": + // A document has operators of its own, not one query per + // index, and generated code declares its fields through + // gensupport's json kind: it is run in PostgreSQL by its + // own test, with the generated fields. + return default: t.Fatalf("no plaintexts for wire kind %q: add a case so %s is tested", typ.Plaintext, typ.Name) } diff --git a/packages/stack-encrypt/fuzz/fuzz_targets/check_record.rs b/packages/stack-encrypt/fuzz/fuzz_targets/check_record.rs index 402c6e441..2bce08130 100644 --- a/packages/stack-encrypt/fuzz/fuzz_targets/check_record.rs +++ b/packages/stack-encrypt/fuzz/fuzz_targets/check_record.rs @@ -33,7 +33,7 @@ use arbitrary::Arbitrary; use libfuzzer_sys::fuzz_target; use stack_encrypt::dynamic::record::{check_record, plan_with, Plan}; use stack_encrypt::dynamic::{ - Output, TargetDescriptor, TargetError, TargetResolver, ValueFrame, ValueKind, + Output, TargetDescriptor, TargetError, TargetOptions, TargetResolver, ValueFrame, ValueKind, }; use stack_encrypt::{ Encrypt, KeysetCipher, Label, PassthroughFrame, Pending, SealedValue, StackCipher, @@ -86,6 +86,7 @@ impl TargetResolver for Fixed { fn encrypt<'a, K: 'static>( &self, _: &str, + _: Option<&TargetOptions>, _: &'a KeysetCipher<'_, K>, _: &Label, _: ValueFrame, From deb4578c993fd6f8e89b5598dddb7a4dfa933fb1 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 01:55:24 +1100 Subject: [PATCH 04/21] test(stack-encrypt): pin the JSON text scan, wipe_value and TargetOptions::json The mutants gate found the integer pre-scan's index arithmetic unpinned (a scan that left a string early passed while the number it reached fitted, and several replacements hung). The scan now walks slices with no arithmetic to mutate, and tests pin a long number inside a string, a number after an escaped quote, an unterminated string, a lone sign and a malformed token. The visitor drops visit_string, visit_none and visit_some, which serde_json never calls. wipe_value and TargetOptions::json get unit tests. Two survivors are excluded with their reason: the strict visitor's expecting and visit_unit (equivalent), and JsonPlaintext's Drop (no safe code can read a value after it is dropped; what it runs is wipe_value, now pinned). Refs #1347 Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- .cargo/mutants.toml | 11 ++ packages/stack-encrypt/src/dynamic/target.rs | 10 ++ packages/stack-encrypt/src/json/mod.rs | 18 +++ packages/stack-encrypt/src/json/text.rs | 131 +++++++++++-------- 4 files changed, 117 insertions(+), 53 deletions(-) diff --git a/.cargo/mutants.toml b/.cargo/mutants.toml index 0ce84f41f..331962238 100644 --- a/.cargo/mutants.toml +++ b/.cargo/mutants.toml @@ -116,6 +116,17 @@ exclude_re = [ # A document always has a root, so its terms are never empty: `false` is # the value `is_empty` returns. The `true` replacement is still caught. 'stack-encrypt/src/json/derive\.rs:\d+:\d+: replace JsonTerms::is_empty -> bool with false$', + # JSON text at the boundary. The strict visitor's `expecting` writes only + # the "expected ..." half of a serde error's message, and `classify` reads + # nothing of a message but the duplicate-member marker, so writing + # nothing refuses the same text with the same error. Its `visit_unit` + # returns `Value::Null`, which is `Value::default()`. + 'stack-encrypt/src/json/text\.rs:\d+:\d+: replace for StrictVisitor>::(expecting -> fmt::Result|visit_unit -> Result) with Ok\(Default::default\(\)\)$', + # `JsonPlaintext`'s `Drop` wipes the document as it is freed: no safe code + # can read the value once it is dropped, so no test can tell the wipe from + # none. What it runs, `wipe_value`, is pinned by + # `wipe_value_leaves_no_plaintext_in_the_value`. + 'stack-encrypt/src/dynamic/json\.rs:\d+:\d+: replace ::drop with \(\)$', ] # Headroom over the measured baseline before a slow-but-correct mutant is diff --git a/packages/stack-encrypt/src/dynamic/target.rs b/packages/stack-encrypt/src/dynamic/target.rs index 3593c2c3e..3fcf97abe 100644 --- a/packages/stack-encrypt/src/dynamic/target.rs +++ b/packages/stack-encrypt/src/dynamic/target.rs @@ -657,6 +657,16 @@ mod tests { use super::*; use crate::dynamic::test_frame::Tree; + #[test] + fn json_options_are_read_back_as_given() { + let mut options = JsonOptions::new(); + options.sealed_root = false; + let _ = options.ordered.insert("$.age".parse().unwrap()); + let target = TargetOptions::Json(options.clone()); + assert_eq!(target.json(), Some(&options)); + assert_ne!(target.json(), Some(&JsonOptions::default())); + } + fn text_eq() -> TargetDescriptor { TargetDescriptor::new( "TextEq", diff --git a/packages/stack-encrypt/src/json/mod.rs b/packages/stack-encrypt/src/json/mod.rs index fe6539814..24dde3d73 100644 --- a/packages/stack-encrypt/src/json/mod.rs +++ b/packages/stack-encrypt/src/json/mod.rs @@ -928,6 +928,24 @@ impl TryFrom for JsonNode { mod tests { use super::*; + #[test] + fn wipe_value_leaves_no_plaintext_in_the_value() { + let mut value = + serde_json::json!({"name": "Alice", "n": 34, "ok": true, "list": ["x", 1.5]}); + wipe_value(&mut value); + assert_eq!(value, serde_json::json!({}), "the containers are emptied"); + let mut text = serde_json::json!("secret"); + wipe_value(&mut text); + assert_eq!( + text, + serde_json::json!(""), + "a string is overwritten and cleared" + ); + let mut number = serde_json::json!(42); + wipe_value(&mut number); + assert_eq!(number, serde_json::json!(0)); + } + #[test] fn the_header_round_trips_its_frozen_layout() { let header = JsonHeader::new(Uuid::from_u128(7), [9; 16], vec![1, 2, 3], true); diff --git a/packages/stack-encrypt/src/json/text.rs b/packages/stack-encrypt/src/json/text.rs index 95fe863a9..5600d051c 100644 --- a/packages/stack-encrypt/src/json/text.rs +++ b/packages/stack-encrypt/src/json/text.rs @@ -127,53 +127,68 @@ fn classify(error: serde_json::Error) -> JsonTextError { /// of each number token with no fraction or exponent. Syntax errors are /// left to the parser. fn check_integers(text: &[u8]) -> Result<(), JsonTextError> { - let mut i = 0; - while i < text.len() { - match text[i] { - b'"' => { - i += 1; - while i < text.len() && text[i] != b'"' { - // An escape takes the next byte with it, so `\"` does - // not end the string. - i += if text[i] == b'\\' { 2 } else { 1 }; - } - i += 1; - } + let mut rest = text; + while let Some((&first, after_first)) = rest.split_first() { + rest = match first { + b'"' => after_string(after_first), b'-' | b'0'..=b'9' => { - let start = i; - i += 1; - while i < text.len() - && matches!(text[i], b'0'..=b'9' | b'.' | b'e' | b'E' | b'+' | b'-') - { - i += 1; - } - let token = &text[start..i]; - if token.iter().any(|b| matches!(b, b'.' | b'e' | b'E')) { - continue; - } - // ASCII by construction; a malformed token is the parser's. - let Ok(token) = std::str::from_utf8(token) else { - continue; - }; - let digits = token.strip_prefix('-').unwrap_or(token); - if digits.is_empty() || !digits.bytes().all(|b| b.is_ascii_digit()) { - continue; - } - let fits = if token.starts_with('-') { - token.parse::().is_ok() - } else { - token.parse::().is_ok() - }; - if !fits { - return Err(JsonTextError::IntegerRange); - } + // The token is the first byte and every number byte after it. + let more = after_first + .iter() + .take_while(|b| matches!(b, b'0'..=b'9' | b'.' | b'e' | b'E' | b'+' | b'-')) + .count(); + check_integer(&rest[..=more])?; + &after_first[more..] } - _ => i += 1, - } + _ => after_first, + }; } Ok(()) } +/// The text after a string whose opening quote is already consumed. An +/// escape takes the next byte with it, so `\"` does not end the string; an +/// unterminated string runs to the end, for the parser to refuse. +fn after_string(mut rest: &[u8]) -> &[u8] { + loop { + rest = match rest { + [b'"', after @ ..] => return after, + [b'\\', _, after @ ..] | [_, after @ ..] => after, + [] => return rest, + }; + } +} + +/// Refuse one number token that is an integer literal out of range. A +/// token with a fraction or an exponent is a float; a malformed one is the +/// parser's. +fn check_integer(token: &[u8]) -> Result<(), JsonTextError> { + if token.iter().any(|b| matches!(b, b'.' | b'e' | b'E')) { + return Ok(()); + } + let (negative, digits) = match token.split_first() { + Some((b'-', digits)) => (true, digits), + _ => (false, token), + }; + if digits.is_empty() || !digits.iter().all(u8::is_ascii_digit) { + return Ok(()); + } + // ASCII digits by the check above, so always UTF-8. + let Ok(text) = std::str::from_utf8(token) else { + return Ok(()); + }; + let fits = if negative { + text.parse::().is_ok() + } else { + text.parse::().is_ok() + }; + if fits { + Ok(()) + } else { + Err(JsonTextError::IntegerRange) + } +} + /// A `Value`, refusing an object that names a member twice. struct Strict; @@ -215,22 +230,10 @@ impl<'de> Visitor<'de> for StrictVisitor { Ok(Value::String(v.to_owned())) } - fn visit_string(self, v: String) -> Result { - Ok(Value::String(v)) - } - fn visit_unit(self) -> Result { Ok(Value::Null) } - fn visit_none(self) -> Result { - Ok(Value::Null) - } - - fn visit_some>(self, d: D) -> Result { - Strict.deserialize(d) - } - fn visit_seq>(self, mut seq: A) -> Result { let mut items = Vec::new(); while let Some(item) = seq.next_element_seed(Strict)? { @@ -297,6 +300,28 @@ mod tests { ); } + #[test] + fn a_long_number_inside_a_string_is_text_and_one_after_it_is_checked() { + // Thirty digits: any suffix of them is out of range too, so a scan + // that left the string early would refuse it. + let text = br#"["123456789012345678901234567890", "\\", 99999999999999999999]"#; + assert_eq!(parse_text(text), Err(JsonTextError::IntegerRange)); + assert_eq!( + parse_text(br#"["123456789012345678901234567890", "\\"]"#).unwrap(), + json!(["123456789012345678901234567890", "\\"]) + ); + } + + #[test] + fn an_unterminated_string_or_a_lone_sign_is_the_parsers_to_refuse() { + for text in [&b"\"abc"[..], b"\"ab\\", b"-", b"[1-2]", b"--1"] { + assert!( + matches!(parse_text(text), Err(JsonTextError::Syntax { .. })), + "{text:?}" + ); + } + } + #[test] fn a_number_inside_a_string_is_text() { assert_eq!( From ffd421236752bd4284f3f72ba969ac991bb8795d Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 03:11:04 +1100 Subject: [PATCH 05/21] fix(stack-encrypt,eql,go): close the json-tree data-plan review findings - json::parse_text is the engine's own strict parser, held to serde_json by a differential property test and a new fuzz:json-text target, so no heap block it frees holds the document (serde_json's scratch buffer and partial documents on refusal did); to_text sizes its buffer first. A heap-spy test watches the boundary. - check_source reads a json field's document by the engine's rules, so a malformed one is refused before any key, as encrypt refuses it. - A present "options": null is refused rather than read as defaults. - eql-bindings' query() refuses a document by name (DocumentQuery); a test pins the engine's and eql-bindings' rules for which targets take JSON options to agree. - Go: the per-type PostgreSQL test no longer counts EncryptedJson; opened JSON numbers compare as text; EQLGoName decodes a rune; stashgen names the missing document support instead of a kind mismatch. - Docs: ValueKind's break under Breaking, the Go CHANGELOG no longer points at internal/record, the generated eql doc sentence reads. Refs #1347 Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- .github/workflows/fuzz.yml | 2 + docs/fuzzing.md | 1 + languages/golang/CHANGELOG.md | 5 +- languages/golang/cmd/stashgen/main_test.go | 16 +- languages/golang/encrypt/eql/eql_gen.go | 4 +- languages/golang/encrypt/eql_postgres_test.go | 12 +- languages/golang/encrypt/json_tree_test.go | 18 +- languages/golang/stashgen/engine.go | 4 + languages/golang/stashgen/read.go | 3 +- packages/eql/crates/eql-bindings/CHANGELOG.md | 4 +- .../eql-bindings/src/encryption/targets.rs | 12 + packages/eql/crates/eql-codegen/src/go_eql.rs | 12 +- .../tests/encryption/tests/heap_hygiene.rs | 41 ++ .../encryption/tests/json_tree_data_plan.rs | 49 ++ packages/stack-encrypt/CHANGELOG.md | 19 +- packages/stack-encrypt/fuzz/Cargo.toml | 7 + .../corpus/json_text/valid-escapes-numbers | 1 + .../fuzz/corpus/json_text/valid-object | 1 + .../fuzz/fuzz_targets/json_text.rs | 32 + packages/stack-encrypt/src/dynamic/record.rs | 67 +- packages/stack-encrypt/src/json/text.rs | 692 +++++++++++++++--- packages/stack-encrypt/tasks.toml | 9 + 22 files changed, 865 insertions(+), 146 deletions(-) create mode 100644 packages/stack-encrypt/fuzz/corpus/json_text/valid-escapes-numbers create mode 100644 packages/stack-encrypt/fuzz/corpus/json_text/valid-object create mode 100644 packages/stack-encrypt/fuzz/fuzz_targets/json_text.rs diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml index ad76faf70..7b60acd54 100644 --- a/.github/workflows/fuzz.yml +++ b/.github/workflows/fuzz.yml @@ -79,6 +79,7 @@ jobs: - { task: "fuzz:check-record", slug: check-record } - { task: "fuzz:plan-build", slug: plan-build } - { task: "fuzz:json-wire", slug: json-wire } + - { task: "fuzz:json-text", slug: json-text } steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: @@ -128,6 +129,7 @@ jobs: - { task: "fuzz:check-record", slug: check-record, dir: packages/stack-encrypt, target: check_record } - { task: "fuzz:plan-build", slug: plan-build, dir: packages/stack-encrypt, target: plan_build } - { task: "fuzz:json-wire", slug: json-wire, dir: packages/stack-encrypt, target: json_wire_decode } + - { task: "fuzz:json-text", slug: json-text, dir: packages/stack-encrypt, target: json_text } steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: diff --git a/docs/fuzzing.md b/docs/fuzzing.md index ca130747b..406dfa52c 100644 --- a/docs/fuzzing.md +++ b/docs/fuzzing.md @@ -28,6 +28,7 @@ Current targets: | `fuzz:term-decode` | `stack-encrypt` | `term_decode` | the SEM term decoders (`EqualityTerm`, `MatchTerms`, `OreTerm`, `OpeTerm` `from_bytes`) | | `fuzz:check-record` | `stack-encrypt` | `check_record` | `dynamic::record::check_record` over an `Arbitrary`-derived plan and tree, against a model of the record rules (structure-aware) | | `fuzz:plan-build` | `stack-encrypt` | `plan_build` | `plan::FieldsBuilder::build` over `Arbitrary`-derived untrusted names (context, fields, identities); a built field's label must render and parse back, and passthrough names are never refused (structure-aware) | +| `fuzz:json-text` | `stack-encrypt` | `json_text` | `json::parse_text`, the boundary parser for a language SDK's JSON document, against serde_json: it reads what serde_json reads to the same value, refuses beyond it only repeated members and out-of-range integers, and what it reads writes back to itself | | `fuzz:json-wire` | `stack-encrypt` | `json_wire_decode` | the JSON index's wire decoders (`JsonDocument`, `JsonNode`, `JsonProjection`, `JsonLeafQuery` from JSON text; `JsonHeader::from_bytes`), over base64 tags and base85 ciphertexts; an accepted document or node must round-trip through its wire form | Each target is a few lines — `libfuzzer-sys` hands a `&str` (or `&[u8]` diff --git a/languages/golang/CHANGELOG.md b/languages/golang/CHANGELOG.md index 23f7e95d4..7d51d6e8b 100644 --- a/languages/golang/CHANGELOG.md +++ b/languages/golang/CHANGELOG.md @@ -16,9 +16,8 @@ listed under **Breaking**. Releases are tagged `languages/golang/vX.Y.Z`. `eql.EncryptedJSON` (`public.eql_v4_json_tree`), its containment needle `eql.EncryptedJSONQuery`, its ordering operand `eql.EncryptedJSONLeafQuery` and its entry `eql.EncryptedJSONEntry`, - generated from the EQL catalog. A plan built by hand can store one now - (`record.JSON`, `record.Field.TargetOptions`); `stashgen` support - follows. + generated from the EQL catalog. The engine stores one from the + module's own plans; `stashgen` does not generate a field of it yet. - **Decimals, dates and timestamps take an order index (#1236).** An `encrypt.Decimal`, `encrypt.Date` or `time.Time` field takes `ore` and diff --git a/languages/golang/cmd/stashgen/main_test.go b/languages/golang/cmd/stashgen/main_test.go index 298a67a2b..42fbf9687 100644 --- a/languages/golang/cmd/stashgen/main_test.go +++ b/languages/golang/cmd/stashgen/main_test.go @@ -214,6 +214,17 @@ func TestRunAsksTheEmbeddedEngine(t *testing.T) { if !strings.Contains(stderr.String(), "User.Age") || !strings.Contains(stderr.String(), "TextEq seals a string, and int64 is int") { t.Fatalf("stderr = %q", stderr.String()) } + // A document type is refused by what is missing, not by a kind + // mismatch no Go type could fix. + document := strings.Replace(userSource, "Email string `stash:\"email,encrypt_into=TextEq\"`", "Profile []byte `stash:\"profile,encrypt_into=EncryptedJson\"`", 1) + dir = writeModule(t, map[string]string{"model.go": document}) + stderr.Reset() + if code := run([]string{"-type", "User"}, dir, &stdout, &stderr, stashgen.GuestEngine); code != 1 { + t.Fatalf("exit %d, want 1\n%s", code, stderr.String()) + } + if !strings.Contains(stderr.String(), "User.Profile") || !strings.Contains(stderr.String(), "EncryptedJson seals a json document, and stashgen does not generate a field of it yet") { + t.Fatalf("stderr = %q", stderr.String()) + } // A context of two segments leaves an EQL field no column: refused by // name, before the engine is asked, and nothing written. deep := strings.Replace(userSource, "context=users", "context=app/users", 1) @@ -260,7 +271,10 @@ func TestEQLGoNameAgreesWithTheGeneratedPackage(t *testing.T) { t.Fatal("the rule renamed nothing: the Json family is spelled JSON") } if stashgen.EQLGoName("Json") != "JSON" || stashgen.EQLGoName("TextEq") != "TextEq" || stashgen.EQLGoName("SteVecQuery") != "SteVecQuery" || - stashgen.EQLGoName("EncryptedJsonLeafQuery") != "EncryptedJSONLeafQuery" || stashgen.EQLGoName("Jsonify") != "Jsonify" { + stashgen.EQLGoName("EncryptedJsonLeafQuery") != "EncryptedJSONLeafQuery" || stashgen.EQLGoName("Jsonify") != "Jsonify" || + // A letter after the word is read as a character, as Rust reads it, + // not as its first byte. + stashgen.EQLGoName("Jsonécrit") != "Jsonécrit" || stashgen.EQLGoName("JsonÉcrit") != "JSONÉcrit" { t.Fatal("the one rule: the word Json is JSON, every other name is itself") } } diff --git a/languages/golang/encrypt/eql/eql_gen.go b/languages/golang/encrypt/eql/eql_gen.go index b8407bb23..9d5b67460 100644 --- a/languages/golang/encrypt/eql/eql_gen.go +++ b/languages/golang/encrypt/eql/eql_gen.go @@ -2254,7 +2254,7 @@ func (v *EncryptedJSONQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// EncryptedJSONLeafQuery is the json_tree family's the ordering operand at a path, compared with an entry `->` extracts: the value eql_v4.query_json_tree_leaf holds. +// EncryptedJSONLeafQuery is the json_tree family's ordering operand at a path, compared with an entry `->` extracts: the value eql_v4.query_json_tree_leaf holds. type EncryptedJSONLeafQuery []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -2277,7 +2277,7 @@ func (v *EncryptedJSONLeafQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// EncryptedJSONEntry is the json_tree family's one node of a document, as `->` and the walks return it: the value public.eql_v4_json_tree_entry holds. +// EncryptedJSONEntry is the json_tree family's entry: one node of a document, as `->` and the walks return it: the value public.eql_v4_json_tree_entry holds. type EncryptedJSONEntry []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. diff --git a/languages/golang/encrypt/eql_postgres_test.go b/languages/golang/encrypt/eql_postgres_test.go index 9f9d4d677..fcdc81d65 100644 --- a/languages/golang/encrypt/eql_postgres_test.go +++ b/languages/golang/encrypt/eql_postgres_test.go @@ -113,6 +113,12 @@ func TestEQLTypesRoundTripAndQueryInPostgres(t *testing.T) { if !typ.Producible { continue } + // A document has operators of its own, not one query per index, + // so it is not run here or counted: it is run in PostgreSQL with + // the generated fields that store it. + if typ.Plaintext == "json" { + continue + } covered++ t.Run(typ.Name, func(t *testing.T) { h := eqlHarness{t: t, ctx: ctx, cipher: cipher, conn: conn, typ: typ} @@ -148,12 +154,6 @@ func TestEQLTypesRoundTripAndQueryInPostgres(t *testing.T) { ds[i] = d } runEQLType(h, gensupport.Decimal, ds, nil) - case "json": - // A document has operators of its own, not one query per - // index, and generated code declares its fields through - // gensupport's json kind: it is run in PostgreSQL by its - // own test, with the generated fields. - return default: t.Fatalf("no plaintexts for wire kind %q: add a case so %s is tested", typ.Plaintext, typ.Name) } diff --git a/languages/golang/encrypt/json_tree_test.go b/languages/golang/encrypt/json_tree_test.go index b0a3db4c2..0b14d796a 100644 --- a/languages/golang/encrypt/json_tree_test.go +++ b/languages/golang/encrypt/json_tree_test.go @@ -1,6 +1,7 @@ package encrypt_test import ( + "bytes" "context" "encoding/json" "errors" @@ -72,11 +73,13 @@ func TestJSONDocumentsRoundTripThroughTheGuest(t *testing.T) { if !ok { t.Fatalf("row %d opened as %T, not bytes", i, row["profile"]) } - var got, want any - if err := json.Unmarshal(text, &got); err != nil { + // Numbers are compared as their text (UseNumber), so an integer + // past 2^53 that came back as the nearest float would differ. + got, err := decodeNumbers(text) + if err != nil { t.Fatalf("row %d: %v", i, err) } - _ = json.Unmarshal([]byte(jsonDocuments[i]), &want) + want, _ := decodeNumbers([]byte(jsonDocuments[i])) if !reflect.DeepEqual(got, want) { t.Fatalf("row %d: opened %s, want %s", i, text, jsonDocuments[i]) } @@ -105,3 +108,12 @@ func TestJSONTextThatIsNotADocumentIsErrEncoding(t *testing.T) { } } } + +// decodeNumbers decodes JSON text with each number kept as its text. +func decodeNumbers(text []byte) (any, error) { + dec := json.NewDecoder(bytes.NewReader(text)) + dec.UseNumber() + var v any + err := dec.Decode(&v) + return v, err +} diff --git a/languages/golang/stashgen/engine.go b/languages/golang/stashgen/engine.go index b99a197ae..6af725617 100644 --- a/languages/golang/stashgen/engine.go +++ b/languages/golang/stashgen/engine.go @@ -225,6 +225,10 @@ func lowerDeclaration(d Declaration, eqlTypes []EQLType) (*record.Plan, error) { // intoKind takes every built-in integer except into a // 128-bit kind, which gensupport does not convert to. return nil, &FieldError{Type: d.Type, Field: f.GoName, Reason: fmt.Sprintf("%s seals %s, and the built-in integer %s is not converted to it", f.EQLType, eqlType.wire, f.GoType)} + } else if eqlType.wire == record.JSON { + // A document's kind is no Go scalar's: say what is + // missing, not a kind mismatch. + return nil, &FieldError{Type: d.Type, Field: f.GoName, Reason: fmt.Sprintf("%s seals a json document, and stashgen does not generate a field of it yet", f.EQLType)} } else { // The engine checks the kind too (se_plan_check refuses a // "type" other than the EQL type's plaintext); saying it diff --git a/languages/golang/stashgen/read.go b/languages/golang/stashgen/read.go index 3b5602fb3..bc35ba64d 100644 --- a/languages/golang/stashgen/read.go +++ b/languages/golang/stashgen/read.go @@ -5,6 +5,7 @@ import ( "go/types" "strings" "unicode" + "unicode/utf8" "github.com/cipherstash/stack/languages/golang/internal/record" "golang.org/x/tools/go/packages" @@ -167,7 +168,7 @@ func EQLGoName(name string) string { } after := rest[at+4:] out.WriteString(rest[:at]) - if after == "" || unicode.IsUpper(rune(after[0])) { + if next, _ := utf8.DecodeRuneInString(after); after == "" || unicode.IsUpper(next) { out.WriteString("JSON") } else { out.WriteString("Json") diff --git a/packages/eql/crates/eql-bindings/CHANGELOG.md b/packages/eql/crates/eql-bindings/CHANGELOG.md index 355f6b817..e82026411 100644 --- a/packages/eql/crates/eql-bindings/CHANGELOG.md +++ b/packages/eql/crates/eql-bindings/CHANGELOG.md @@ -18,7 +18,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 and seals the document's JSON text through `v4::json_tree::JsonTree` under them; `decrypt` opens it back to its text. A value that is not a JSON document is `TargetError::Json`, options on a type that takes none - `TargetError::Options`. `v4::all()` lists `JsonTreeDocument` and + `TargetError::Options`, and a one-plaintext `query` of a document + `TargetError::DocumentQuery` (its operands are the engine's, by JSON + operator). `v4::all()` lists `JsonTreeDocument` and `v4::all_query()` lists `JsonTreeQuery` and `JsonTreeLeafQuery`. - **The EQL v4 `json_tree` family's types, `v4::json_tree`** (`stack-encrypt` diff --git a/packages/eql/crates/eql-bindings/src/encryption/targets.rs b/packages/eql/crates/eql-bindings/src/encryption/targets.rs index 4bf82085c..132ed7096 100644 --- a/packages/eql/crates/eql-bindings/src/encryption/targets.rs +++ b/packages/eql/crates/eql-bindings/src/encryption/targets.rs @@ -201,6 +201,15 @@ pub enum TargetError { /// The type's name. name: &'static str, }, + /// The type stores a JSON document, which is queried by operator + /// (`contains`, `eq_at`, an ordering at a path, a read) from named + /// inputs, not from one plaintext: Stack Encrypt derives those + /// operands itself (`stack_encrypt::dynamic::query_term`). + #[error("{name} is queried by a JSON operator, not by one plaintext")] + DocumentQuery { + /// The type's name. + name: &'static str, + }, /// A query asked through an index the type does not carry /// ([`Target::indexes`]), so it has no term to derive. #[error("{name} carries no {index} index, so no query is answered through it")] @@ -490,6 +499,9 @@ pub fn query<'a, K: 'static>( return Err(error); } carries(target, index)?; + if document(name).is_some() { + return Err(TargetError::DocumentQuery { name: target.name }); + } let column = Identifier::from_label(context)?; query_named(name, index, keyset, column, plaintext) } diff --git a/packages/eql/crates/eql-codegen/src/go_eql.rs b/packages/eql/crates/eql-codegen/src/go_eql.rs index 1a5a7c37f..c6b3608ad 100644 --- a/packages/eql/crates/eql-codegen/src/go_eql.rs +++ b/packages/eql/crates/eql-codegen/src/go_eql.rs @@ -184,9 +184,11 @@ fn document_types(out: &mut String) { let what = match domain.role { DocumentRole::Stored | DocumentRole::Query => continue, DocumentRole::LeafQuery => { - "the ordering operand at a path, compared with an entry `->` extracts" + "ordering operand at a path, compared with an entry `->` extracts" + } + DocumentRole::Entry => { + "entry: one node of a document, as `->` and the walks return it" } - DocumentRole::Entry => "one node of a document, as `->` and the walks return it", }; let go = go_name(domain.name); let _ = write!( @@ -253,6 +255,12 @@ mod tests { assert_eq!(go_name("EncryptedJson"), "EncryptedJSON"); assert_eq!(go_name("EncryptedJsonLeafQuery"), "EncryptedJSONLeafQuery"); assert_eq!(go_name("Jsonify"), "Jsonify", "a word, not a prefix"); + assert_eq!( + go_name("Jsonécrit"), + "Jsonécrit", + "the next character, not byte" + ); + assert_eq!(go_name("JsonÉcrit"), "JSONÉcrit"); } #[test] diff --git a/packages/eql/tests/encryption/tests/heap_hygiene.rs b/packages/eql/tests/encryption/tests/heap_hygiene.rs index ef7818cbe..5473fd042 100644 --- a/packages/eql/tests/encryption/tests/heap_hygiene.rs +++ b/packages/eql/tests/encryption/tests/heap_hygiene.rs @@ -241,6 +241,47 @@ fn a_text_target_leaves_no_plaintext_on_the_heap() { } } +/// A JSON document at the engine's boundary: the parser a binding's text +/// goes through (`json::parse_text`), with the marker in a string that has +/// escapes (decoded, not borrowed), as a key, and past 128 bytes; a refused +/// document, whose part built so far is wiped; and the text an opened +/// document goes back as (`dynamic::json_frame`), written into a buffer +/// that never reallocates. +/// +/// The seal and open themselves are the JSON index's +/// (`stack_encrypt::json`), which is not held to this rule yet: its path +/// derivation frees key and leaf plaintext unwiped, so they are not +/// watched here. +#[test] +fn a_document_leaves_no_plaintext_on_the_heap_at_the_boundary() { + let mark = marked("doc"); + let document = format!( + r#"{{"note": "{mark}\n\"quoted\"", "{mark}": [1.5, "{mark}{pad}"], "n": 12345678901234567}}"#, + pad = "x".repeat(160) + ); + let value = watched("parse_text", || { + stack_encrypt::json::parse_text(document.as_bytes()).unwrap() + }); + watched("json_frame", || { + let frame = stack_encrypt::dynamic::json_frame(&value).unwrap(); + drop(frame); + }); + watched("wipe_value", || { + let mut value = value; + stack_encrypt::json::wipe_value(&mut value); + }); + for refused in [ + format!(r#"{{"a": "{mark}\n", "b": ["{mark}"], "a": 1}}"#), + format!(r#"["{mark}\n", {{"{mark}": [true, "{mark}"#), + format!(r#"{{"{mark}": "{mark}", "x": 99999999999999999999}}"#), + ] { + watched("parse_text refusal", || { + assert!(stack_encrypt::json::parse_text(refused.as_bytes()).is_err()); + }); + } + drop(document); +} + /// The spy itself: a plain `String` holding the marker, freed while /// watched, fails the watch. #[test] diff --git a/packages/eql/tests/encryption/tests/json_tree_data_plan.rs b/packages/eql/tests/encryption/tests/json_tree_data_plan.rs index 0b348cb9a..c9f2e5d96 100644 --- a/packages/eql/tests/encryption/tests/json_tree_data_plan.rs +++ b/packages/eql/tests/encryption/tests/json_tree_data_plan.rs @@ -334,3 +334,52 @@ async fn text_that_is_not_a_document_is_refused_before_any_key() { "no key was requested" ); } + +/// A document is queried by a JSON operator, which the engine derives +/// from named inputs; eql-bindings' one-plaintext query refuses it by +/// name rather than falling through its generated dispatch. +#[tokio::test] +async fn a_document_has_no_one_plaintext_query() { + let (cipher, _) = common::cipher().await; + let keyset = cipher.default_keyset(); + let label: Label = "users/profile".parse().unwrap(); + let error = eql_targets::query( + "EncryptedJson", + &stack_encrypt::target::IndexSpec::Json(JsonOptions::default()), + &keyset, + &label, + frame("{}"), + ) + .err() + .expect("refused"); + assert!( + matches!( + error, + eql_targets::TargetError::DocumentQuery { + name: "EncryptedJson" + } + ), + "{error}" + ); +} + +/// Which targets take the JSON index's options is decided twice: the +/// engine gives them to a target whose descriptor carries the `json` +/// index, and eql-bindings seals a target of a document family with them +/// and refuses them on any other. The two must name the same producible +/// targets, or a write the engine configures is one eql-bindings refuses. +#[tokio::test] +async fn the_engine_and_the_table_agree_on_which_targets_take_json_options() { + let (cipher, _) = common::cipher().await; + let keyset = cipher.default_keyset(); + let label: Label = "users/profile".parse().unwrap(); + let options = dynamic::TargetOptions::Json(JsonOptions::default()); + for target in eql_targets::targets().iter().filter(|t| t.producible) { + let engine = target.indexes.contains(&"json"); + let refused = matches!( + eql_targets::encrypt_with(target.name, Some(&options), &keyset, &label, frame(true)), + Err(eql_targets::TargetError::Options { .. }) + ); + assert_eq!(engine, !refused, "{}", target.name); + } +} diff --git a/packages/stack-encrypt/CHANGELOG.md b/packages/stack-encrypt/CHANGELOG.md index e7f4a9614..4e0f2d07f 100644 --- a/packages/stack-encrypt/CHANGELOG.md +++ b/packages/stack-encrypt/CHANGELOG.md @@ -13,6 +13,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `encrypt(name, options: Option<&TargetOptions>, keyset, label, plaintext)`. A resolver implemented outside this crate adds the parameter; one that resolves no document type ignores it. +- **`dynamic::ValueKind` gains `Json`, and `ValueKind::ALL` holds 21 + kinds** (#1349). The enum is not `#[non_exhaustive]`, so an exhaustive + `match` on it, or a `[ValueKind; 20]` binding of `ALL`, stops compiling. ### Fixed @@ -43,11 +46,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 (`Reason::JsonNeedsTarget`) and a sealed null on one. The document crosses as UTF-8 JSON text in a text or bytes leaf and opens back to its text in a bytes leaf (`dynamic::read_json`, `dynamic::json_frame`). - `json::parse_text` is the one parser: it refuses an object that names a - member twice and an integer literal outside `i64::MIN ..= u64::MAX` - (which serde_json would read as the nearest `f64`), as well as text - that is not JSON, each with a code (`JsonTextError::code`) and a - position, never a byte of the text. `TargetError::Json` carries the code + `json::parse_text` is the one parser, the engine's own (held to + serde_json's by a differential property test and the `fuzz:json-text` + target), so no heap block it frees holds the document: it refuses an + object that names a member twice and an integer literal outside + `i64::MIN ..= u64::MAX` (which serde_json would read as the nearest + `f64`), as well as text that is not JSON, each with a code + (`JsonTextError::code`) and a position, never a byte of the text. + `json::to_text` sizes its buffer before it writes. `check_source` reads + a document by these rules too, so a malformed one is refused before + any key, as `encrypt` refuses it. A present `"options": null` is + refused (`Reason::InvalidTarget`), not read as the defaults. `TargetError::Json` carries the code when a resolver refuses a document. `json::wipe_value` wipes a parsed document in place. `ValueKind::holds` says whether a value of one kind is a value of a declared one (`json` holds text and bytes). diff --git a/packages/stack-encrypt/fuzz/Cargo.toml b/packages/stack-encrypt/fuzz/Cargo.toml index 6f66c1508..8b47f231a 100644 --- a/packages/stack-encrypt/fuzz/Cargo.toml +++ b/packages/stack-encrypt/fuzz/Cargo.toml @@ -72,5 +72,12 @@ test = false doc = false bench = false +[[bin]] +name = "json_text" +path = "fuzz_targets/json_text.rs" +test = false +doc = false +bench = false + [workspace] resolver = "2" diff --git a/packages/stack-encrypt/fuzz/corpus/json_text/valid-escapes-numbers b/packages/stack-encrypt/fuzz/corpus/json_text/valid-escapes-numbers new file mode 100644 index 000000000..db0fc2be3 --- /dev/null +++ b/packages/stack-encrypt/fuzz/corpus/json_text/valid-escapes-numbers @@ -0,0 +1 @@ +["\"\\\/\b\f\n\r\t", "é", "😀", -0, -0.0, 1.5e3, 18446744073709551615, -9223372036854775808] \ No newline at end of file diff --git a/packages/stack-encrypt/fuzz/corpus/json_text/valid-object b/packages/stack-encrypt/fuzz/corpus/json_text/valid-object new file mode 100644 index 000000000..041f2bea5 --- /dev/null +++ b/packages/stack-encrypt/fuzz/corpus/json_text/valid-object @@ -0,0 +1 @@ +{"name": "Alice", "age": 34, "tags": ["a", "b"], "n": null, "ok": true} \ No newline at end of file diff --git a/packages/stack-encrypt/fuzz/fuzz_targets/json_text.rs b/packages/stack-encrypt/fuzz/fuzz_targets/json_text.rs new file mode 100644 index 000000000..96f1d0984 --- /dev/null +++ b/packages/stack-encrypt/fuzz/fuzz_targets/json_text.rs @@ -0,0 +1,32 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; +use serde_json::Value; +use stack_encrypt::json::{parse_text, to_text, JsonTextError}; + +// Fuzz the boundary parser a language SDK's JSON document goes through +// (`json::parse_text`): the text is the caller's, so it must never panic. +// It is the engine's own parser, so it is held to serde_json's, which it +// must agree with: +// +// - whatever it reads, serde_json reads to the same value; +// - whatever serde_json reads and it refuses is a repeated member or an +// integer literal out of the 64-bit range, never a syntax error; +// - what it reads writes back (`to_text`) to text it reads to the same +// value. +fuzz_target!(|bytes: &[u8]| { + let ours = parse_text(bytes); + let theirs = serde_json::from_slice::(bytes); + match (&ours, &theirs) { + (Ok(ours), Ok(theirs)) => assert_eq!(ours, theirs, "the two parsers read different values"), + (Ok(_), Err(_)) => panic!("parse_text read text serde_json refuses"), + (Err(JsonTextError::Syntax { .. }), Ok(_)) => { + panic!("parse_text refused as not JSON text serde_json reads") + } + _ => {} + } + if let Ok(value) = ours { + let text = to_text(&value).expect("a value writes"); + assert_eq!(parse_text(&text).ok(), Some(value), "a value does not read back"); + } +}); diff --git a/packages/stack-encrypt/src/dynamic/record.rs b/packages/stack-encrypt/src/dynamic/record.rs index 28bb57e25..83da9741b 100644 --- a/packages/stack-encrypt/src/dynamic/record.rs +++ b/packages/stack-encrypt/src/dynamic/record.rs @@ -1421,7 +1421,7 @@ enum TargetWire { #[serde(deny_unknown_fields)] struct ConfiguredTarget { name: String, - #[serde(default)] + #[serde(default, deserialize_with = "present")] options: Option, } @@ -2392,6 +2392,17 @@ fn check_field(value: &ValueFrame, field: &FieldPlan) -> Result<(), Error> { return Err(misfit(Reason::FieldType)); } } + // A document is read by the engine's rules here, before any key, so a + // check refuses what encrypt refuses, and no resolver has to. + if field.field_type == Some(ValueKind::Json) { + if let Err(error) = super::read_json(value) { + return Err(Error::Target(TargetError::Json { + name: field.name.clone(), + target: field.target().unwrap_or_default().to_owned(), + reason: error.code(), + })); + } + } // A target field's value is sealed by the type's own plan, which // refuses a passthrough as any ciphertext does. let sealed = field.is_target() || field.outputs.contains(&Output::Ciphertext); @@ -6637,6 +6648,28 @@ mod tests { &FakeEql, )); assert_eq!(error.reason(), Some(Reason::TargetOptions), "{error}"); + // A present `"options"` holds an object: a null is refused, + // never read as no options. + for target in [TEXT_EQ, "EncryptedJson"] { + let error = refused(plan_with( + obj(vec![( + "email", + obj(vec![ + ("context", label("email")), + ( + "target", + obj(vec![("name", s(target)), ("options", Tree::Null)]), + ), + ]), + )]), + &FakeEql, + )); + assert_eq!( + error.reason(), + Some(Reason::InvalidTarget), + "{target}: {error}" + ); + } let error = refused(plan_with( obj(vec![( "email", @@ -6678,6 +6711,38 @@ mod tests { assert!(plan_with(obj(vec![("profile", spec)]), &FakeEql).is_ok()); } + /// A document that is not one by the engine's rules is refused by + /// the check, before any key, with the code encrypt gives it. + #[test] + fn the_check_reads_a_document_by_the_engines_rules() { + let mut spec = target_spec(label("profile"), "EncryptedJson"); + if let Tree::Object(entries) = &mut spec { + entries.push(("null".to_string(), s("sql"))); + } + let plan = plan_with(obj(vec![("profile", spec)]), &FakeEql).unwrap(); + for (text, code) in [ + (r#"{"secret": 1, "secret": 2}"#, "json_duplicate_key"), + (r#"{"secret""#, "json_syntax"), + ("18446744073709551616", "json_integer_range"), + ] { + let error = check_source(obj(vec![("profile", s(text))]), &plan).unwrap_err(); + let Error::Target(TargetError::Json { + name, + target, + reason, + }) = &error + else { + panic!("{text}: {error}"); + }; + assert_eq!( + (name.as_str(), target.as_str(), *reason), + ("profile", "EncryptedJson", code) + ); + assert!(!error.to_string().contains("secret"), "{error}"); + } + assert!(check_source(obj(vec![("profile", s(r#"{"a": [1, null]}"#))]), &plan).is_ok()); + } + #[tokio::test] async fn is_sealed_by_the_resolver_in_the_records_one_request() { let cipher = cipher().await; diff --git a/packages/stack-encrypt/src/json/text.rs b/packages/stack-encrypt/src/json/text.rs index 5600d051c..1157c19e4 100644 --- a/packages/stack-encrypt/src/json/text.rs +++ b/packages/stack-encrypt/src/json/text.rs @@ -10,7 +10,8 @@ //! `i64::MIN ..= u64::MAX` is refused ([`JsonTextError::IntegerRange`]) //! rather than read as the nearest `f64`, which would seal a different //! number than the text names. A literal with a fraction or an exponent -//! is the nearest binary64 (serde_json's `float_roundtrip` parser), so +//! is the nearest binary64 (correctly rounded, as serde_json's +//! `float_roundtrip` parser reads it), so //! `1e400` is refused as not JSON this engine reads, and `0.1` is the //! double nearest to it. //! - **An object names each member once.** A repeated key is refused @@ -19,7 +20,8 @@ //! first) and the sealed document would depend on which ran. //! - **Text is valid Unicode**: invalid UTF-8 and an unpaired surrogate //! escape (`"\ud800"`) are refused as not JSON. -//! - **Nesting** is limited by the parser (128 levels) and by the index +//! - **Nesting** is limited by the parser (127 levels of containers, +//! serde_json's limit) and by the index //! ([`TermError::JsonTooDeep`](crate::sem::TermError::JsonTooDeep)). //! //! What comes back out ([`to_text`]) is JSON text serde_json writes: @@ -31,10 +33,8 @@ //! Every error names a class and, for a syntax error, a line and column, //! and never a byte of the text. -use std::fmt; - -use serde::de::{DeserializeSeed, Deserializer, Error as _, MapAccess, SeqAccess, Visitor}; use serde_json::{Map, Number, Value}; +use zeroize::Zeroize; /// Why JSON text was refused at the boundary. Each says what kind of /// mistake it was and where, never what the text held. @@ -76,12 +76,15 @@ impl JsonTextError { } } -/// The marker a duplicate key is raised with inside the parse, told apart -/// from serde_json's own errors by its text, which nothing else writes. -const DUPLICATE: &str = "\u{0}stack-encrypt: duplicate member"; - /// Parse a document from UTF-8 JSON text by the rules in the module docs. /// +/// The parser is the engine's own, not serde_json's deserializer, so that +/// no heap block it frees holds plaintext: each string is decoded into a +/// buffer sized for it before it is read, so it never grows; a container's +/// buffer is wiped before it is replaced by a larger one; numbers are read +/// from the text in place; and on a refusal the part of the document built +/// so far is wiped ([`wipe_value`](super::wipe_value)) before it drops. +/// /// # Errors /// /// [`JsonTextError`]: the text is not JSON, an object repeats a member, @@ -97,162 +100,413 @@ const DUPLICATE: &str = "\u{0}stack-encrypt: duplicate member"; /// # Ok::<(), JsonTextError>(()) /// ``` pub fn parse_text(text: &[u8]) -> Result { - check_integers(text)?; - let mut de = serde_json::Deserializer::from_slice(text); - let value = Strict.deserialize(&mut de).map_err(classify)?; - de.end().map_err(classify)?; + let mut parser = Parser { text, at: 0 }; + let mut value = parser.value(MAX_DEPTH)?; + parser.skip_whitespace(); + if parser.at != text.len() { + super::wipe_value(&mut value); + return Err(parser.syntax()); + } Ok(value) } -/// The document as JSON text: what opening returns to a binding. +/// The document as JSON text: what opening returns to a binding. The +/// buffer is sized from the text's length before it is written, so it is +/// never reallocated and no freed block holds part of it. /// /// # Errors /// /// None in practice: every `Value` serializes. The `Result` is serde_json's. pub fn to_text(value: &Value) -> Result, serde_json::Error> { - serde_json::to_vec(value) + let mut counter = Counter(0); + serde_json::to_writer(&mut counter, value)?; + let mut text = Vec::with_capacity(counter.0); + serde_json::to_writer(&mut text, value)?; + Ok(text) } -fn classify(error: serde_json::Error) -> JsonTextError { - let (line, column) = (error.line(), error.column()); - if error.to_string().starts_with(DUPLICATE) { - JsonTextError::DuplicateKey { line, column } - } else { - JsonTextError::Syntax { line, column } +/// A writer that counts the bytes it is given and keeps none. +struct Counter(usize); + +impl std::io::Write for Counter { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.0 += buf.len(); + Ok(buf.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) } } -/// Refuse an integer literal serde_json would read as the nearest `f64`: -/// one outside `i64::MIN ..= u64::MAX`. A lexical scan, outside strings, -/// of each number token with no fraction or exponent. Syntax errors are -/// left to the parser. -fn check_integers(text: &[u8]) -> Result<(), JsonTextError> { - let mut rest = text; - while let Some((&first, after_first)) = rest.split_first() { - rest = match first { - b'"' => after_string(after_first), - b'-' | b'0'..=b'9' => { - // The token is the first byte and every number byte after it. - let more = after_first - .iter() - .take_while(|b| matches!(b, b'0'..=b'9' | b'.' | b'e' | b'E' | b'+' | b'-')) - .count(); - check_integer(&rest[..=more])?; - &after_first[more..] - } - _ => after_first, - }; - } - Ok(()) +/// How deep containers nest: serde_json's limit, so a document either +/// parser reads, the other reads too. +const MAX_DEPTH: usize = 127; + +struct Parser<'t> { + text: &'t [u8], + at: usize, } -/// The text after a string whose opening quote is already consumed. An -/// escape takes the next byte with it, so `\"` does not end the string; an -/// unterminated string runs to the end, for the parser to refuse. -fn after_string(mut rest: &[u8]) -> &[u8] { - loop { - rest = match rest { - [b'"', after @ ..] => return after, - [b'\\', _, after @ ..] | [_, after @ ..] => after, - [] => return rest, - }; +impl Parser<'_> { + /// The line and column of the byte at `at`, from 1, as a syntax error. + fn syntax(&self) -> JsonTextError { + let (line, column) = self.place(self.at); + JsonTextError::Syntax { line, column } } -} -/// Refuse one number token that is an integer literal out of range. A -/// token with a fraction or an exponent is a float; a malformed one is the -/// parser's. -fn check_integer(token: &[u8]) -> Result<(), JsonTextError> { - if token.iter().any(|b| matches!(b, b'.' | b'e' | b'E')) { - return Ok(()); - } - let (negative, digits) = match token.split_first() { - Some((b'-', digits)) => (true, digits), - _ => (false, token), - }; - if digits.is_empty() || !digits.iter().all(u8::is_ascii_digit) { - return Ok(()); - } - // ASCII digits by the check above, so always UTF-8. - let Ok(text) = std::str::from_utf8(token) else { - return Ok(()); - }; - let fits = if negative { - text.parse::().is_ok() - } else { - text.parse::().is_ok() - }; - if fits { - Ok(()) - } else { - Err(JsonTextError::IntegerRange) + fn place(&self, at: usize) -> (usize, usize) { + let before = &self.text[..at.min(self.text.len())]; + let line = before.iter().filter(|&&b| b == b'\n').count() + 1; + let start = before + .iter() + .rposition(|&b| b == b'\n') + .map_or(0, |n| n + 1); + (line, before.len() - start + 1) } -} -/// A `Value`, refusing an object that names a member twice. -struct Strict; + fn peek(&self) -> Option { + self.text.get(self.at).copied() + } -impl<'de> DeserializeSeed<'de> for Strict { - type Value = Value; - fn deserialize>(self, d: D) -> Result { - d.deserialize_any(StrictVisitor) + fn skip_whitespace(&mut self) { + while matches!(self.peek(), Some(b' ' | b'\t' | b'\n' | b'\r')) { + self.at += 1; + } } -} -struct StrictVisitor; + /// Consume `byte`, or refuse. + fn expect(&mut self, byte: u8) -> Result<(), JsonTextError> { + if self.peek() == Some(byte) { + self.at += 1; + Ok(()) + } else { + Err(self.syntax()) + } + } -impl<'de> Visitor<'de> for StrictVisitor { - type Value = Value; + /// One value, after any whitespace, nested at most `depth` more levels. + fn value(&mut self, depth: usize) -> Result { + self.skip_whitespace(); + match self.peek() { + Some(b'{') => self.object(depth), + Some(b'[') => self.array(depth), + Some(b'"') => self.string().map(Value::String), + Some(b'-' | b'0'..=b'9') => self.number(), + Some(b't') => self.literal(b"true", Value::Bool(true)), + Some(b'f') => self.literal(b"false", Value::Bool(false)), + Some(b'n') => self.literal(b"null", Value::Null), + _ => Err(self.syntax()), + } + } - fn expecting(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str("a JSON value") + fn literal(&mut self, word: &[u8], value: Value) -> Result { + if self.text[self.at..].starts_with(word) { + self.at += word.len(); + Ok(value) + } else { + Err(self.syntax()) + } } - fn visit_bool(self, v: bool) -> Result { - Ok(Value::Bool(v)) + /// A container's opening byte, consumed, at one level less depth. + fn open(&mut self, depth: usize) -> Result { + let Some(inner) = depth.checked_sub(1) else { + return Err(self.syntax()); + }; + self.at += 1; + Ok(inner) } - fn visit_i64(self, v: i64) -> Result { - Ok(Value::Number(v.into())) + fn array(&mut self, depth: usize) -> Result { + let inner = self.open(depth)?; + let mut items = Vec::new(); + let result = self.items(inner, &mut items); + let mut value = Value::Array(items); + match result { + Ok(()) => Ok(value), + Err(error) => { + super::wipe_value(&mut value); + Err(error) + } + } } - fn visit_u64(self, v: u64) -> Result { - Ok(Value::Number(v.into())) + fn items(&mut self, depth: usize, items: &mut Vec) -> Result<(), JsonTextError> { + self.skip_whitespace(); + if self.peek() == Some(b']') { + self.at += 1; + return Ok(()); + } + loop { + push_wiping(items, self.value(depth)?); + self.skip_whitespace(); + match self.peek() { + Some(b',') => self.at += 1, + Some(b']') => { + self.at += 1; + return Ok(()); + } + _ => return Err(self.syntax()), + } + } } - fn visit_f64(self, v: f64) -> Result { - Number::from_f64(v) - .map(Value::Number) - .ok_or_else(|| E::custom("a number that is not finite")) + fn object(&mut self, depth: usize) -> Result { + let inner = self.open(depth)?; + let mut members = Map::new(); + let result = self.members(inner, &mut members); + let mut value = Value::Object(members); + match result { + Ok(()) => Ok(value), + Err(error) => { + super::wipe_value(&mut value); + Err(error) + } + } } - fn visit_str(self, v: &str) -> Result { - Ok(Value::String(v.to_owned())) + fn members( + &mut self, + depth: usize, + members: &mut Map, + ) -> Result<(), JsonTextError> { + self.skip_whitespace(); + if self.peek() == Some(b'}') { + self.at += 1; + return Ok(()); + } + loop { + self.skip_whitespace(); + if self.peek() != Some(b'"') { + return Err(self.syntax()); + } + let mut key = self.string()?; + if members.contains_key(&key) { + key.zeroize(); + // The place of the name's closing quote, as serde_json + // reports it. + let (line, column) = self.place(self.at - 1); + return Err(JsonTextError::DuplicateKey { line, column }); + } + self.skip_whitespace(); + if let Err(error) = self.expect(b':') { + key.zeroize(); + return Err(error); + } + match self.value(depth) { + Ok(value) => { + let _ = members.insert(key, value); + } + Err(error) => { + key.zeroize(); + return Err(error); + } + } + self.skip_whitespace(); + match self.peek() { + Some(b',') => self.at += 1, + Some(b'}') => { + self.at += 1; + return Ok(()); + } + _ => return Err(self.syntax()), + } + } } - fn visit_unit(self) -> Result { - Ok(Value::Null) + /// A string, from its opening quote. Its decoded text is never longer + /// than its raw text, so the buffer is sized to that before anything is + /// written and never grows; on a refusal it is wiped. + fn string(&mut self) -> Result { + self.at += 1; + let raw = self.text[self.at..] + .iter() + .scan(false, |escaped, &b| { + let end = b == b'"' && !*escaped; + *escaped = !*escaped && b == b'\\'; + Some(end) + }) + .position(|end| end) + .unwrap_or(self.text.len() - self.at); + let mut out = String::with_capacity(raw); + match self.decode(&mut out) { + Ok(()) => Ok(out), + Err(error) => { + out.zeroize(); + Err(error) + } + } } - fn visit_seq>(self, mut seq: A) -> Result { - let mut items = Vec::new(); - while let Some(item) = seq.next_element_seed(Strict)? { - items.push(item); + fn decode(&mut self, out: &mut String) -> Result<(), JsonTextError> { + loop { + // A run of plain bytes, copied as one checked UTF-8 slice. + let run = self.text[self.at..] + .iter() + .take_while(|&&b| b != b'"' && b != b'\\' && b >= 0x20) + .count(); + let plain = std::str::from_utf8(&self.text[self.at..self.at + run]).map_err(|e| { + self.at += e.valid_up_to(); + self.syntax() + })?; + out.push_str(plain); + self.at += run; + match self.peek() { + Some(b'"') => { + self.at += 1; + return Ok(()); + } + Some(b'\\') => { + self.at += 1; + let c = self.escape()?; + out.push(c); + } + // A control character, or the end of the text. + _ => return Err(self.syntax()), + } } - Ok(Value::Array(items)) } - fn visit_map>(self, mut map: A) -> Result { - let mut members = Map::new(); - while let Some(key) = map.next_key::()? { - if members.contains_key(&key) { - return Err(A::Error::custom(DUPLICATE)); + /// The character an escape names, after its backslash. + fn escape(&mut self) -> Result { + let c = match self.peek() { + Some(b'"') => '"', + Some(b'\\') => '\\', + Some(b'/') => '/', + Some(b'b') => '\u{8}', + Some(b'f') => '\u{c}', + Some(b'n') => '\n', + Some(b'r') => '\r', + Some(b't') => '\t', + Some(b'u') => { + self.at += 1; + return self.unicode(); } - let value = map.next_value_seed(Strict)?; - let _ = members.insert(key, value); + _ => return Err(self.syntax()), + }; + self.at += 1; + Ok(c) + } + + /// `\uXXXX`, after its `u`: one code point, or a surrogate pair. An + /// unpaired surrogate is refused. + fn unicode(&mut self) -> Result { + let high = self.hex4()?; + let code = if (0xD800..0xDC00).contains(&high) { + if !self.text[self.at..].starts_with(b"\\u") { + return Err(self.syntax()); + } + self.at += 2; + let low = self.hex4()?; + if !(0xDC00..0xE000).contains(&low) { + return Err(self.syntax()); + } + 0x10000 + ((high - 0xD800) << 10) + (low - 0xDC00) + } else { + high + }; + char::from_u32(code).ok_or_else(|| self.syntax()) + } + + fn hex4(&mut self) -> Result { + let digits = self + .text + .get(self.at..self.at + 4) + .ok_or_else(|| self.syntax())?; + let mut code = 0; + for &d in digits { + let Some(v) = char::from(d).to_digit(16) else { + return Err(self.syntax()); + }; + code = code * 16 + v; + } + self.at += 4; + Ok(code) + } + + /// A number, read from the text in place: an integer literal is a + /// signed or unsigned 64-bit integer or refused, and one with a + /// fraction or an exponent is the nearest binary64 (std's parser, + /// correctly rounded, as serde_json's `float_roundtrip` is). + fn number(&mut self) -> Result { + let start = self.at; + let negative = self.peek() == Some(b'-'); + if negative { + self.at += 1; + } + match self.peek() { + Some(b'0') => self.at += 1, + Some(b'1'..=b'9') => self.digits(), + _ => return Err(self.syntax()), + } + let mut float = false; + if self.peek() == Some(b'.') { + self.at += 1; + self.some_digits()?; + float = true; + } + if matches!(self.peek(), Some(b'e' | b'E')) { + self.at += 1; + if matches!(self.peek(), Some(b'+' | b'-')) { + self.at += 1; + } + self.some_digits()?; + float = true; } - Ok(Value::Object(members)) + // ASCII by the grammar just read. + let token = std::str::from_utf8(&self.text[start..self.at]).map_err(|_| self.syntax())?; + let number = if float { + token + .parse::() + .ok() + .and_then(Number::from_f64) + .ok_or_else(|| self.syntax())? + } else if !negative { + Number::from( + token + .parse::() + .map_err(|_| JsonTextError::IntegerRange)?, + ) + } else { + match token + .parse::() + .map_err(|_| JsonTextError::IntegerRange)? + { + // serde_json reads `-0` as the float negative zero. + 0 => Number::from_f64(-0.0).ok_or_else(|| self.syntax())?, + n => Number::from(n), + } + }; + Ok(Value::Number(number)) } + + fn digits(&mut self) { + while matches!(self.peek(), Some(b'0'..=b'9')) { + self.at += 1; + } + } + + /// At least one digit. + fn some_digits(&mut self) -> Result<(), JsonTextError> { + if !matches!(self.peek(), Some(b'0'..=b'9')) { + return Err(self.syntax()); + } + self.digits(); + Ok(()) + } +} + +/// Push onto `items`, wiping the old buffer when it is replaced by a larger +/// one: the values it held were moved, but their bytes (a number, the +/// pointer and length of a string) stay in the freed block otherwise. +fn push_wiping(items: &mut Vec, value: Value) { + if items.len() == items.capacity() { + let mut grown = Vec::with_capacity((items.capacity() * 2).max(4)); + grown.append(items); + items.spare_capacity_mut().zeroize(); + *items = grown; + } + items.push(value); } #[cfg(test)] @@ -370,4 +624,200 @@ mod tests { let text = to_text(&value).unwrap(); assert_eq!(parse_text(&text).unwrap(), value); } + + #[test] + fn escapes_and_unicode_decode_as_serde_json_reads_them() { + let text = r#"["\"\\\/\b\f\n\r\t", "\u00e9\u4e2d", "\ud83d\ude00", "é中😀", "a\u0000b"]"# + .as_bytes(); + assert_eq!( + parse_text(text).unwrap(), + serde_json::from_slice::(text).unwrap() + ); + for bad in [ + &br#""\ud800""#[..], + br#""\ud800A""#, + br#""\udc00""#, + br#""\x""#, + br#""\u12""#, + br#""\u12g4""#, + b"\"a\nb\"", + b"\"\x01\"", + b"\"\\", + ] { + assert!( + matches!(parse_text(bad), Err(JsonTextError::Syntax { .. })), + "{bad:?}" + ); + } + } + + #[test] + fn numbers_read_as_serde_json_reads_them() { + for text in [ + &b"0"[..], + b"-0", + b"-0.0", + b"1.5e3", + b"1E-2", + b"2e+2", + b"0.1", + b"-9223372036854775808", + b"18446744073709551615", + b"123456789012345678.5", + b"5e-324", + b"1e-400", + ] { + let ours = parse_text(text).unwrap(); + let theirs: Value = serde_json::from_slice(text).unwrap(); + assert_eq!(ours, theirs, "{}", String::from_utf8_lossy(text)); + assert_eq!( + ours.as_f64().map(f64::to_bits), + theirs.as_f64().map(f64::to_bits), + "{}", + String::from_utf8_lossy(text) + ); + } + for bad in [ + &b"01"[..], + b"-", + b"1.", + b".5", + b"1e", + b"1e+", + b"+1", + b"0x1", + b"--1", + b"1.e2", + ] { + assert!( + matches!(parse_text(bad), Err(JsonTextError::Syntax { .. })), + "{bad:?}" + ); + } + } + + #[test] + fn nesting_stops_at_serde_jsons_limit() { + let deep = |n: usize| [vec![b'['; n], vec![b']'; n]].concat(); + assert!(parse_text(&deep(127)).is_ok()); + assert!(serde_json::from_slice::(&deep(127)).is_ok()); + assert!(matches!( + parse_text(&deep(128)), + Err(JsonTextError::Syntax { .. }) + )); + assert!(serde_json::from_slice::(&deep(128)).is_err()); + let objects = |n: usize| [b"{\"a\":".repeat(n), b"1".to_vec(), vec![b'}'; n]].concat(); + assert!(parse_text(&objects(127)).is_ok()); + assert!(parse_text(&objects(128)).is_err()); + } + + #[test] + fn literals_and_structure_are_strict() { + for bad in [ + &b"tru"[..], + b"nul", + b"falsey", + b"[1,]", + b"[1 2]", + b"{\"a\" 1}", + b"{\"a\":1,}", + b"{1:1}", + b"{\"a\":1 \"b\":2}", + b"[", + b"{", + b"{\"a\":", + b"]", + ] { + assert!( + matches!(parse_text(bad), Err(JsonTextError::Syntax { .. })), + "{bad:?}" + ); + } + assert_eq!( + parse_text(b" \t\r\n[ true , false , null ] ").unwrap(), + json!([true, false, null]) + ); + assert_eq!(parse_text(b"{ }").unwrap(), json!({})); + assert_eq!(parse_text(b"[ ]").unwrap(), json!([])); + } + + #[test] + fn a_syntax_error_names_its_line_and_column() { + assert_eq!( + parse_text(b"[1,\n x]"), + Err(JsonTextError::Syntax { line: 2, column: 3 }) + ); + assert_eq!( + parse_text(b"?"), + Err(JsonTextError::Syntax { line: 1, column: 1 }) + ); + } + + #[test] + fn long_arrays_grow_and_keep_their_order() { + let many: Vec = (0..100).collect(); + let text = serde_json::to_vec(&many).unwrap(); + assert_eq!(parse_text(&text).unwrap(), json!(many)); + } + + #[test] + fn to_text_writes_exactly_its_length() { + let value = json!({"name": "Alice \"A\" \u{e9}", "n": [1.5, -2, 18446744073709551615u64]}); + let text = to_text(&value).unwrap(); + assert_eq!(text.len(), text.capacity(), "sized before it is written"); + assert_eq!(text, serde_json::to_vec(&value).unwrap()); + } + + mod differential { + use super::*; + use proptest::prelude::*; + + fn document() -> impl Strategy { + let leaf = prop_oneof![ + Just(Value::Null), + any::().prop_map(Value::Bool), + any::().prop_map(|n| json!(n)), + any::().prop_map(|n| json!(n)), + any::() + .prop_filter("finite", |f| f.is_finite()) + .prop_map(|f| json!(f)), + any::().prop_map(Value::String), + ]; + leaf.prop_recursive(4, 32, 6, |inner| { + prop_oneof![ + prop::collection::vec(inner.clone(), 0..6).prop_map(Value::Array), + prop::collection::btree_map(any::(), inner, 0..6) + .prop_map(|m| Value::Object(m.into_iter().collect())), + ] + }) + } + + proptest! { + /// Every document serde_json writes, compact or pretty, reads + /// back to the same value, numbers to the same bits. + #[test] + fn reads_what_serde_json_writes(value in document()) { + for text in [serde_json::to_vec(&value).unwrap(), serde_json::to_vec_pretty(&value).unwrap()] { + prop_assert_eq!(&parse_text(&text).unwrap(), &value); + } + } + + /// Over text near JSON: whatever this parser reads, serde_json + /// reads to the same value; whatever serde_json reads and this + /// one refuses is a repeated member or an integer out of range. + #[test] + fn agrees_with_serde_json_on_arbitrary_text(text in r#"[\[\]{}",:0-9eE.+\- tfrunlasa\\\\u]{0,24}"#) { + let ours = parse_text(text.as_bytes()); + let theirs = serde_json::from_slice::(text.as_bytes()); + match (ours, theirs) { + (Ok(ours), Ok(theirs)) => prop_assert_eq!(ours, theirs), + (Ok(ours), Err(_)) => prop_assert!(false, "read {} that serde_json refuses", ours), + (Err(JsonTextError::Syntax { .. }), Ok(theirs)) => { + prop_assert!(false, "refused {:?} that serde_json reads as {}", text, theirs) + } + _ => {} + } + } + } + } } diff --git a/packages/stack-encrypt/tasks.toml b/packages/stack-encrypt/tasks.toml index fb96b0ea5..bb337a20c 100644 --- a/packages/stack-encrypt/tasks.toml +++ b/packages/stack-encrypt/tasks.toml @@ -72,6 +72,15 @@ description = "Fuzz stack-encrypt's JSON index wire decoders: JsonDocument, Json dir = "{{config_root}}/packages/stack-encrypt" run = "mise x --env test -- cargo +nightly fuzz run json_wire_decode --sanitizer none --target $(rustc -vV | sed -n 's/^host: //p') -- -max_total_time=60" +# Fuzz the boundary parser a language SDK's JSON document goes through +# (`json::parse_text`), the engine's own: it must never panic, and it is held +# to serde_json, reading what serde_json reads to the same value and refusing +# only repeated members and out-of-range integers beyond it. +["fuzz:json-text"] +description = "Fuzz stack-encrypt's JSON text parser (json::parse_text) against serde_json (libFuzzer, nightly, 60s default)" +dir = "{{config_root}}/packages/stack-encrypt" +run = "mise x --env test -- cargo +nightly fuzz run json_text --sanitizer none --target $(rustc -vV | sed -n 's/^host: //p') -- -max_total_time=60" + # Rustdoc with warnings as errors: a broken intra-doc link or a rustdoc # warning fails the build. Doc *examples* are `test:doc:stack-encrypt`. Both run # with all features so nothing feature-gated goes unchecked; the root `doc` From d36fb1683ea67c3ef1aeb2829fc6f91291eb6069 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 05:20:34 +1100 Subject: [PATCH 06/21] fix(stack-encrypt): move the JSON text parser by re-slicing, and pin its wipes A local mutants run timed out on the parser's index arithmetic: every `at += 1` replaced by `at *= 1` loops forever, which the gate counts as a survivor. The parser now holds the unread text as a slice and moves by re-slicing, so a mutation changes what it reads rather than hanging. Tests pin a string's buffer to its raw length, an array's growth from four by doubling, and a container closed by the other bracket. JsonPlaintext's Drop is no longer excluded from mutation: heap_hygiene's new a_json_document_read_from_a_frame_leaves_no_plaintext_on_the_heap fails when it is a no-op. The strict-visitor exclusion goes with the visitor. Refs #1347 Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- .cargo/mutants.toml | 11 - packages/stack-encrypt/src/json/text.rs | 251 +++++++++++-------- packages/stack-encrypt/tests/heap_hygiene.rs | 19 ++ 3 files changed, 171 insertions(+), 110 deletions(-) diff --git a/.cargo/mutants.toml b/.cargo/mutants.toml index 331962238..0ce84f41f 100644 --- a/.cargo/mutants.toml +++ b/.cargo/mutants.toml @@ -116,17 +116,6 @@ exclude_re = [ # A document always has a root, so its terms are never empty: `false` is # the value `is_empty` returns. The `true` replacement is still caught. 'stack-encrypt/src/json/derive\.rs:\d+:\d+: replace JsonTerms::is_empty -> bool with false$', - # JSON text at the boundary. The strict visitor's `expecting` writes only - # the "expected ..." half of a serde error's message, and `classify` reads - # nothing of a message but the duplicate-member marker, so writing - # nothing refuses the same text with the same error. Its `visit_unit` - # returns `Value::Null`, which is `Value::default()`. - 'stack-encrypt/src/json/text\.rs:\d+:\d+: replace for StrictVisitor>::(expecting -> fmt::Result|visit_unit -> Result) with Ok\(Default::default\(\)\)$', - # `JsonPlaintext`'s `Drop` wipes the document as it is freed: no safe code - # can read the value once it is dropped, so no test can tell the wipe from - # none. What it runs, `wipe_value`, is pinned by - # `wipe_value_leaves_no_plaintext_in_the_value`. - 'stack-encrypt/src/dynamic/json\.rs:\d+:\d+: replace ::drop with \(\)$', ] # Headroom over the measured baseline before a slow-but-correct mutant is diff --git a/packages/stack-encrypt/src/json/text.rs b/packages/stack-encrypt/src/json/text.rs index 1157c19e4..a11445c08 100644 --- a/packages/stack-encrypt/src/json/text.rs +++ b/packages/stack-encrypt/src/json/text.rs @@ -100,10 +100,10 @@ impl JsonTextError { /// # Ok::<(), JsonTextError>(()) /// ``` pub fn parse_text(text: &[u8]) -> Result { - let mut parser = Parser { text, at: 0 }; + let mut parser = Parser { text, rest: text }; let mut value = parser.value(MAX_DEPTH)?; parser.skip_whitespace(); - if parser.at != text.len() { + if !parser.rest.is_empty() { super::wipe_value(&mut value); return Err(parser.syntax()); } @@ -142,45 +142,49 @@ impl std::io::Write for Counter { /// parser reads, the other reads too. const MAX_DEPTH: usize = 127; +/// The parser's place: the whole text, for reporting a position, and the +/// part not read yet. It moves by re-slicing `rest`, never by index +/// arithmetic. struct Parser<'t> { text: &'t [u8], - at: usize, + rest: &'t [u8], } -impl Parser<'_> { - /// The line and column of the byte at `at`, from 1, as a syntax error. +impl<'t> Parser<'t> { + /// A syntax error at the first byte not read. fn syntax(&self) -> JsonTextError { - let (line, column) = self.place(self.at); + self.syntax_at(self.rest) + } + + /// A syntax error at the first byte of `at`, a tail of the text. + fn syntax_at(&self, at: &[u8]) -> JsonTextError { + let (line, column) = self.place(at); JsonTextError::Syntax { line, column } } - fn place(&self, at: usize) -> (usize, usize) { - let before = &self.text[..at.min(self.text.len())]; + /// The line and column, from 1, of the first byte of `at`, a tail of + /// the text (the end, for an empty one). + fn place(&self, at: &[u8]) -> (usize, usize) { + let before = &self.text[..self.text.len() - at.len()]; let line = before.iter().filter(|&&b| b == b'\n').count() + 1; - let start = before - .iter() - .rposition(|&b| b == b'\n') - .map_or(0, |n| n + 1); - (line, before.len() - start + 1) + let column = before.iter().rev().take_while(|&&b| b != b'\n').count() + 1; + (line, column) } fn peek(&self) -> Option { - self.text.get(self.at).copied() + self.rest.first().copied() } - fn skip_whitespace(&mut self) { - while matches!(self.peek(), Some(b' ' | b'\t' | b'\n' | b'\r')) { - self.at += 1; + /// Step past the next byte, which the caller has peeked. + fn bump(&mut self) { + if let [_, tail @ ..] = self.rest { + self.rest = tail; } } - /// Consume `byte`, or refuse. - fn expect(&mut self, byte: u8) -> Result<(), JsonTextError> { - if self.peek() == Some(byte) { - self.at += 1; - Ok(()) - } else { - Err(self.syntax()) + fn skip_whitespace(&mut self) { + while let [b' ' | b'\t' | b'\n' | b'\r', tail @ ..] = self.rest { + self.rest = tail; } } @@ -200,11 +204,12 @@ impl Parser<'_> { } fn literal(&mut self, word: &[u8], value: Value) -> Result { - if self.text[self.at..].starts_with(word) { - self.at += word.len(); - Ok(value) - } else { - Err(self.syntax()) + match self.rest.strip_prefix(word) { + Some(tail) => { + self.rest = tail; + Ok(value) + } + None => Err(self.syntax()), } } @@ -213,10 +218,36 @@ impl Parser<'_> { let Some(inner) = depth.checked_sub(1) else { return Err(self.syntax()); }; - self.at += 1; + self.bump(); Ok(inner) } + /// After a container's member: a comma, or `close` to end it. + fn next_or_close(&mut self, close: u8) -> Result { + self.skip_whitespace(); + match self.peek() { + Some(b',') => { + self.bump(); + Ok(true) + } + Some(byte) if byte == close => { + self.bump(); + Ok(false) + } + _ => Err(self.syntax()), + } + } + + /// Whether the container is empty, consuming its `close` if so. + fn empty(&mut self, close: u8) -> bool { + self.skip_whitespace(); + if self.peek() == Some(close) { + self.bump(); + return true; + } + false + } + fn array(&mut self, depth: usize) -> Result { let inner = self.open(depth)?; let mut items = Vec::new(); @@ -232,21 +263,13 @@ impl Parser<'_> { } fn items(&mut self, depth: usize, items: &mut Vec) -> Result<(), JsonTextError> { - self.skip_whitespace(); - if self.peek() == Some(b']') { - self.at += 1; + if self.empty(b']') { return Ok(()); } loop { push_wiping(items, self.value(depth)?); - self.skip_whitespace(); - match self.peek() { - Some(b',') => self.at += 1, - Some(b']') => { - self.at += 1; - return Ok(()); - } - _ => return Err(self.syntax()), + if !self.next_or_close(b']')? { + return Ok(()); } } } @@ -270,9 +293,7 @@ impl Parser<'_> { depth: usize, members: &mut Map, ) -> Result<(), JsonTextError> { - self.skip_whitespace(); - if self.peek() == Some(b'}') { - self.at += 1; + if self.empty(b'}') { return Ok(()); } loop { @@ -280,19 +301,22 @@ impl Parser<'_> { if self.peek() != Some(b'"') { return Err(self.syntax()); } + let name = self.rest; let mut key = self.string()?; if members.contains_key(&key) { key.zeroize(); // The place of the name's closing quote, as serde_json - // reports it. - let (line, column) = self.place(self.at - 1); + // reports it: the last byte the name took. + let quote = &name[name.len() - self.rest.len() - 1..]; + let (line, column) = self.place(quote); return Err(JsonTextError::DuplicateKey { line, column }); } self.skip_whitespace(); - if let Err(error) = self.expect(b':') { + if self.peek() != Some(b':') { key.zeroize(); - return Err(error); + return Err(self.syntax()); } + self.bump(); match self.value(depth) { Ok(value) => { let _ = members.insert(key, value); @@ -302,14 +326,8 @@ impl Parser<'_> { return Err(error); } } - self.skip_whitespace(); - match self.peek() { - Some(b',') => self.at += 1, - Some(b'}') => { - self.at += 1; - return Ok(()); - } - _ => return Err(self.syntax()), + if !self.next_or_close(b'}')? { + return Ok(()); } } } @@ -318,17 +336,8 @@ impl Parser<'_> { /// than its raw text, so the buffer is sized to that before anything is /// written and never grows; on a refusal it is wiped. fn string(&mut self) -> Result { - self.at += 1; - let raw = self.text[self.at..] - .iter() - .scan(false, |escaped, &b| { - let end = b == b'"' && !*escaped; - *escaped = !*escaped && b == b'\\'; - Some(end) - }) - .position(|end| end) - .unwrap_or(self.text.len() - self.at); - let mut out = String::with_capacity(raw); + self.bump(); + let mut out = String::with_capacity(raw_len(self.rest)); match self.decode(&mut out) { Ok(()) => Ok(out), Err(error) => { @@ -341,25 +350,23 @@ impl Parser<'_> { fn decode(&mut self, out: &mut String) -> Result<(), JsonTextError> { loop { // A run of plain bytes, copied as one checked UTF-8 slice. - let run = self.text[self.at..] + let run = self + .rest .iter() .take_while(|&&b| b != b'"' && b != b'\\' && b >= 0x20) .count(); - let plain = std::str::from_utf8(&self.text[self.at..self.at + run]).map_err(|e| { - self.at += e.valid_up_to(); - self.syntax() - })?; + let (plain, tail) = self.rest.split_at(run); + let plain = std::str::from_utf8(plain).map_err(|_| self.syntax())?; out.push_str(plain); - self.at += run; + self.rest = tail; match self.peek() { Some(b'"') => { - self.at += 1; + self.bump(); return Ok(()); } Some(b'\\') => { - self.at += 1; - let c = self.escape()?; - out.push(c); + self.bump(); + out.push(self.escape()?); } // A control character, or the end of the text. _ => return Err(self.syntax()), @@ -379,12 +386,12 @@ impl Parser<'_> { Some(b'r') => '\r', Some(b't') => '\t', Some(b'u') => { - self.at += 1; + self.bump(); return self.unicode(); } _ => return Err(self.syntax()), }; - self.at += 1; + self.bump(); Ok(c) } @@ -393,10 +400,10 @@ impl Parser<'_> { fn unicode(&mut self) -> Result { let high = self.hex4()?; let code = if (0xD800..0xDC00).contains(&high) { - if !self.text[self.at..].starts_with(b"\\u") { + let Some(tail) = self.rest.strip_prefix(b"\\u") else { return Err(self.syntax()); - } - self.at += 2; + }; + self.rest = tail; let low = self.hex4()?; if !(0xDC00..0xE000).contains(&low) { return Err(self.syntax()); @@ -409,10 +416,9 @@ impl Parser<'_> { } fn hex4(&mut self) -> Result { - let digits = self - .text - .get(self.at..self.at + 4) - .ok_or_else(|| self.syntax())?; + let Some((digits, tail)) = self.rest.split_first_chunk::<4>() else { + return Err(self.syntax()); + }; let mut code = 0; for &d in digits { let Some(v) = char::from(d).to_digit(16) else { @@ -420,7 +426,7 @@ impl Parser<'_> { }; code = code * 16 + v; } - self.at += 4; + self.rest = tail; Ok(code) } @@ -429,38 +435,39 @@ impl Parser<'_> { /// fraction or an exponent is the nearest binary64 (std's parser, /// correctly rounded, as serde_json's `float_roundtrip` is). fn number(&mut self) -> Result { - let start = self.at; + let start = self.rest; let negative = self.peek() == Some(b'-'); if negative { - self.at += 1; + self.bump(); } match self.peek() { - Some(b'0') => self.at += 1, + Some(b'0') => self.bump(), Some(b'1'..=b'9') => self.digits(), _ => return Err(self.syntax()), } let mut float = false; if self.peek() == Some(b'.') { - self.at += 1; + self.bump(); self.some_digits()?; float = true; } if matches!(self.peek(), Some(b'e' | b'E')) { - self.at += 1; + self.bump(); if matches!(self.peek(), Some(b'+' | b'-')) { - self.at += 1; + self.bump(); } self.some_digits()?; float = true; } + let token = &start[..start.len() - self.rest.len()]; // ASCII by the grammar just read. - let token = std::str::from_utf8(&self.text[start..self.at]).map_err(|_| self.syntax())?; + let token = std::str::from_utf8(token).map_err(|_| self.syntax_at(start))?; let number = if float { token .parse::() .ok() .and_then(Number::from_f64) - .ok_or_else(|| self.syntax())? + .ok_or_else(|| self.syntax_at(start))? } else if !negative { Number::from( token @@ -473,7 +480,7 @@ impl Parser<'_> { .map_err(|_| JsonTextError::IntegerRange)? { // serde_json reads `-0` as the float negative zero. - 0 => Number::from_f64(-0.0).ok_or_else(|| self.syntax())?, + 0 => Number::from_f64(-0.0).ok_or_else(|| self.syntax_at(start))?, n => Number::from(n), } }; @@ -481,8 +488,8 @@ impl Parser<'_> { } fn digits(&mut self) { - while matches!(self.peek(), Some(b'0'..=b'9')) { - self.at += 1; + while let [b'0'..=b'9', tail @ ..] = self.rest { + self.rest = tail; } } @@ -496,12 +503,26 @@ impl Parser<'_> { } } +/// The length of a string's raw text, from after its opening quote to +/// before its closing one (or to the end, when it has none): an escaped +/// quote does not end it. +fn raw_len(rest: &[u8]) -> usize { + let mut escaped = false; + rest.iter() + .take_while(|&&b| { + let end = b == b'"' && !escaped; + escaped = !escaped && b == b'\\'; + !end + }) + .count() +} + /// Push onto `items`, wiping the old buffer when it is replaced by a larger /// one: the values it held were moved, but their bytes (a number, the /// pointer and length of a string) stay in the freed block otherwise. fn push_wiping(items: &mut Vec, value: Value) { if items.len() == items.capacity() { - let mut grown = Vec::with_capacity((items.capacity() * 2).max(4)); + let mut grown = Vec::with_capacity(items.capacity().saturating_mul(2).max(4)); grown.append(items); items.spare_capacity_mut().zeroize(); *items = grown; @@ -723,6 +744,8 @@ mod tests { b"{\"a\":1,}", b"{1:1}", b"{\"a\":1 \"b\":2}", + b"[1}", + b"{\"a\":1]", b"[", b"{", b"{\"a\":", @@ -768,6 +791,36 @@ mod tests { assert_eq!(text, serde_json::to_vec(&value).unwrap()); } + #[test] + fn a_string_is_decoded_into_a_buffer_its_raw_length() { + // The buffer is sized once, to the raw text between the quotes, so + // it never grows: an escaped quote does not end the count. + for (text, raw) in [ + (&br#""abc""#[..], 3), + (br#""a\"b\\""#, 6), + (br#""\u00e9x""#, 7), + (br#""""#, 0), + ] { + let Value::String(decoded) = parse_text(text).unwrap() else { + panic!("a string") + }; + assert_eq!(decoded.capacity(), raw, "{}", String::from_utf8_lossy(text)); + } + assert_eq!(raw_len(b"ab"), 2, "an unterminated string runs to the end"); + } + + #[test] + fn an_array_grows_by_doubling_from_four() { + let mut items = Vec::new(); + let mut capacities = Vec::new(); + for n in 0..9 { + push_wiping(&mut items, json!(n)); + capacities.push(items.capacity()); + } + assert_eq!(capacities, [4, 4, 4, 4, 8, 8, 8, 8, 16]); + assert_eq!(items, (0..9).map(|n| json!(n)).collect::>()); + } + mod differential { use super::*; use proptest::prelude::*; diff --git a/packages/stack-encrypt/tests/heap_hygiene.rs b/packages/stack-encrypt/tests/heap_hygiene.rs index 0908d3b25..92dbe9bee 100644 --- a/packages/stack-encrypt/tests/heap_hygiene.rs +++ b/packages/stack-encrypt/tests/heap_hygiene.rs @@ -322,6 +322,25 @@ fn verify_leaves_no_plaintext_on_the_heap() { }); } +/// A JSON document a binding hands over is parsed by the engine's own +/// parser and held in a `JsonPlaintext`, which wipes it when it drops: the +/// marker in a key, in a string with an escape (decoded, not borrowed) and +/// past 128 bytes leaves no freed block. A refused document wipes what was +/// built. +#[test] +fn a_json_document_read_from_a_frame_leaves_no_plaintext_on_the_heap() { + let mark = marked("doc"); + let pad = "x".repeat(160); + let document = format!(r#"{{"{mark}": ["{mark}\n{pad}", 1.5], "n": "{mark}"}}"#); + let refused = format!(r#"{{"{mark}": "{mark}\n", "{mark}": 1}}"#); + let (document, refused) = (frame(document.as_str()), frame(refused.as_str())); + watched("read_json", || { + let parsed = stack_encrypt::dynamic::read_json(&document).expect("a document"); + drop(parsed); + assert!(stack_encrypt::dynamic::read_json(&refused).is_err()); + }); +} + /// The spy itself: a plain `String` holding the marker, freed while /// watched, fails the watch. Without this a spy that saw nothing would pass /// every test above. From ddf9ea1eb4bffe3b02261628b8a8b64a3ce44db3 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 05:34:59 +1100 Subject: [PATCH 07/21] fix(stack-encrypt): ask the buffer whether it is full in push_wiping The local mutants run timed out on `len == capacity` replaced by `!=`: every push then doubled the buffer and wiped its ever larger spare capacity. Asking the buffer for its spare capacity leaves nothing there to mutate; the growth test still pins the sizes. Refs #1347 Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- packages/stack-encrypt/src/json/text.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/stack-encrypt/src/json/text.rs b/packages/stack-encrypt/src/json/text.rs index a11445c08..b6b5f6211 100644 --- a/packages/stack-encrypt/src/json/text.rs +++ b/packages/stack-encrypt/src/json/text.rs @@ -521,7 +521,10 @@ fn raw_len(rest: &[u8]) -> usize { /// one: the values it held were moved, but their bytes (a number, the /// pointer and length of a string) stay in the freed block otherwise. fn push_wiping(items: &mut Vec, value: Value) { - if items.len() == items.capacity() { + // Full when there is no spare capacity; asked of the buffer rather + // than compared, so a mutation of the test cannot make every push grow + // it (and wipe ever larger buffers) instead of failing. + if items.spare_capacity_mut().is_empty() { let mut grown = Vec::with_capacity(items.capacity().saturating_mul(2).max(4)); grown.append(items); items.spare_capacity_mut().zeroize(); From ba65680ae8f83ae3af0a315c8c4c24068d788a90 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 05:45:02 +1100 Subject: [PATCH 08/21] test(stack-encrypt,eql): build heap-spy inputs that leave no marked garbage CI's Linux run of EQL's heap_hygiene flagged a freed array buffer that held the marker in its unused slots: bytes another allocation freed before the watch began (the test's own format! growth, or the text-target test running beside it in the same process). The inputs are now built into strings sized before they are written, and EQL's boundary test, whose binary runs its tests as threads of one process, parses strings only; stack-encrypt's, run one test to a process, keeps containers. Refs #1347 Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- .../tests/encryption/tests/heap_hygiene.rs | 35 +++++++++++++------ packages/stack-encrypt/tests/heap_hygiene.rs | 33 +++++++++++++++-- 2 files changed, 55 insertions(+), 13 deletions(-) diff --git a/packages/eql/tests/encryption/tests/heap_hygiene.rs b/packages/eql/tests/encryption/tests/heap_hygiene.rs index 5473fd042..dacf11e88 100644 --- a/packages/eql/tests/encryption/tests/heap_hygiene.rs +++ b/packages/eql/tests/encryption/tests/heap_hygiene.rs @@ -243,10 +243,15 @@ fn a_text_target_leaves_no_plaintext_on_the_heap() { /// A JSON document at the engine's boundary: the parser a binding's text /// goes through (`json::parse_text`), with the marker in a string that has -/// escapes (decoded, not borrowed), as a key, and past 128 bytes; a refused -/// document, whose part built so far is wiped; and the text an opened -/// document goes back as (`dynamic::json_frame`), written into a buffer -/// that never reallocates. +/// escapes (decoded into its own buffer, not borrowed) and past 128 bytes; +/// a refused string, whose buffer is wiped; and the text an opened document +/// goes back as (`dynamic::json_frame`), written into a buffer that never +/// reallocates. +/// +/// The documents are strings: this binary's tests share one process, and a +/// container's buffer (a `Vec` or a `BTreeMap` node) has unused slots that +/// can hold another test's freed bytes. stack-encrypt's `heap_hygiene`, +/// run one test to a process, holds containers to the same rule. /// /// The seal and open themselves are the JSON index's /// (`stack_encrypt::json`), which is not held to this rule yet: its path @@ -255,10 +260,8 @@ fn a_text_target_leaves_no_plaintext_on_the_heap() { #[test] fn a_document_leaves_no_plaintext_on_the_heap_at_the_boundary() { let mark = marked("doc"); - let document = format!( - r#"{{"note": "{mark}\n\"quoted\"", "{mark}": [1.5, "{mark}{pad}"], "n": 12345678901234567}}"#, - pad = "x".repeat(160) - ); + let pad = "x".repeat(160); + let document = joined(&[r#"""#, &mark, r#"\n\"quoted\" "#, &pad, r#"""#]); let value = watched("parse_text", || { stack_encrypt::json::parse_text(document.as_bytes()).unwrap() }); @@ -271,9 +274,8 @@ fn a_document_leaves_no_plaintext_on_the_heap_at_the_boundary() { stack_encrypt::json::wipe_value(&mut value); }); for refused in [ - format!(r#"{{"a": "{mark}\n", "b": ["{mark}"], "a": 1}}"#), - format!(r#"["{mark}\n", {{"{mark}": [true, "{mark}"#), - format!(r#"{{"{mark}": "{mark}", "x": 99999999999999999999}}"#), + joined(&[r#"""#, &mark, r#"\n\x""#]), + joined(&[r#"""#, &mark, r#"\n"#, &pad]), ] { watched("parse_text refusal", || { assert!(stack_encrypt::json::parse_text(refused.as_bytes()).is_err()); @@ -282,6 +284,17 @@ fn a_document_leaves_no_plaintext_on_the_heap_at_the_boundary() { drop(document); } +/// `parts` joined into a string sized for them before it is written, so +/// building a marked input frees no marked block that a later allocation +/// could reuse and a watch then find. +fn joined(parts: &[&str]) -> String { + let mut out = String::with_capacity(parts.iter().map(|p| p.len()).sum()); + for part in parts { + out.push_str(part); + } + out +} + /// The spy itself: a plain `String` holding the marker, freed while /// watched, fails the watch. #[test] diff --git a/packages/stack-encrypt/tests/heap_hygiene.rs b/packages/stack-encrypt/tests/heap_hygiene.rs index 92dbe9bee..6530af083 100644 --- a/packages/stack-encrypt/tests/heap_hygiene.rs +++ b/packages/stack-encrypt/tests/heap_hygiene.rs @@ -331,8 +331,26 @@ fn verify_leaves_no_plaintext_on_the_heap() { fn a_json_document_read_from_a_frame_leaves_no_plaintext_on_the_heap() { let mark = marked("doc"); let pad = "x".repeat(160); - let document = format!(r#"{{"{mark}": ["{mark}\n{pad}", 1.5], "n": "{mark}"}}"#); - let refused = format!(r#"{{"{mark}": "{mark}\n", "{mark}": 1}}"#); + let document = joined(&[ + r#"{""#, + &mark, + r#"": [""#, + &mark, + r"\n", + &pad, + r#"", 1.5], "n": ""#, + &mark, + r#""}"#, + ]); + let refused = joined(&[ + r#"{""#, + &mark, + r#"": ""#, + &mark, + r#"\n", ""#, + &mark, + r#"": 1}"#, + ]); let (document, refused) = (frame(document.as_str()), frame(refused.as_str())); watched("read_json", || { let parsed = stack_encrypt::dynamic::read_json(&document).expect("a document"); @@ -341,6 +359,17 @@ fn a_json_document_read_from_a_frame_leaves_no_plaintext_on_the_heap() { }); } +/// `parts` joined into a string sized for them before it is written, so +/// building a marked input frees no marked block that a later allocation +/// could reuse and a watch then find. +fn joined(parts: &[&str]) -> String { + let mut out = String::with_capacity(parts.iter().map(|p| p.len()).sum()); + for part in parts { + out.push_str(part); + } + out +} + /// The spy itself: a plain `String` holding the marker, freed while /// watched, fails the watch. Without this a spy that saw nothing would pass /// every test above. From acb9e13e26c2817b96c78a990de09a7953e521ac Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Sun, 25 Oct 2026 23:27:27 +0000 Subject: [PATCH 09/21] feat(eql): name the v4 JSON tree domains as v3 names its JSON document EQL v3 stores a searchable JSON document as `public.eql_v3_json_search`, queries it with `eql_v3.query_json` and returns `public.eql_v3_json_entry` from `->`. In v4 the JSON tree scheme implements that document, so its domains take the same names: `public.eql_v4_json_tree` is now `public.eql_v4_json_search`, `public.eql_v4_json_tree_entry` is `public.eql_v4_json_entry`, and the two query operands are `eql_v4.query_json` and `eql_v4.query_json_leaf`. `json_tree` stays the implementation's name: the SQL directory, the Rust structs and the SQL functions keep it. Those names were already taken in the v4 bundle. The SteVec SQL under src/v3/json/ carried no version line, so it rendered into both bundles and gave v4 a SteVec `json_search`, `json_entry` and `query_json` that nothing writes. The five files, and the generated json_entry_ cross files that bind the SteVec entry to each scalar family's query operands, now carry `-- ONLY EQL VERSION: 3`; eql-codegen writes that line into the cross files. The v3 bundle changes only by those 23 comment lines. With the SteVec SQL gone from v4, the v4 target table drops its SteVecDocument row, which was never producible, and the generated Go package drops the SteVecDocument and SteVecQuery types with it. Both bundles still install side by side: every public domain each owns, the other owns under its own prefix, and both lints report no error. Refs #1347, #1348 Claude-Session: https://claude.ai/code/session_01HLuZxys1NDRmAkv3uhnuoJ --- languages/golang/encrypt/checker.go | 2 +- languages/golang/encrypt/eql/eql_gen.go | 66 +-- .../eql-bindings/src/encryption/targets.rs | 6 +- .../crates/eql-bindings/src/v4/json_tree.rs | 24 +- .../eql/crates/eql-bindings/src/v4/targets.rs | 23 +- .../eql/crates/eql-codegen/src/generate.rs | 47 +- packages/eql/crates/eql-codegen/src/go_eql.rs | 11 +- .../eql/crates/eql-codegen/src/targets.rs | 76 +-- packages/eql/crates/eql-domains/src/lib.rs | 10 +- packages/eql/crates/eql-domains/src/tests.rs | 2 +- packages/eql/docs/README.md | 2 +- .../eql/docs/reference/json-tree-leakage.md | 2 +- packages/eql/src/v3/json/aggregates.sql | 1 + packages/eql/src/v3/json/blockers.sql | 1 + packages/eql/src/v3/json/functions.sql | 1 + packages/eql/src/v3/json/operators.sql | 1 + packages/eql/src/v3/json/types.sql | 1 + packages/eql/src/v3/json_tree/blockers.sql | 484 +++++++++--------- packages/eql/src/v3/json_tree/functions.sql | 84 +-- packages/eql/src/v3/json_tree/operators.sql | 154 +++--- packages/eql/src/v3/json_tree/types.sql | 38 +- .../bigint/json_entry_bigint_functions.sql | 1 + .../bigint/json_entry_bigint_operators.sql | 1 + .../date/json_entry_date_functions.sql | 1 + .../date/json_entry_date_operators.sql | 1 + .../double/json_entry_double_functions.sql | 1 + .../double/json_entry_double_operators.sql | 1 + .../integer/json_entry_integer_functions.sql | 1 + .../integer/json_entry_integer_operators.sql | 1 + .../numeric/json_entry_numeric_functions.sql | 1 + .../numeric/json_entry_numeric_operators.sql | 1 + .../real/json_entry_real_functions.sql | 1 + .../real/json_entry_real_operators.sql | 1 + .../json_entry_smallint_functions.sql | 1 + .../json_entry_smallint_operators.sql | 1 + .../text/json_entry_text_functions.sql | 1 + .../text/json_entry_text_operators.sql | 1 + .../json_entry_timestamp_functions.sql | 1 + .../json_entry_timestamp_operators.sql | 1 + .../eql/tests/encryption/tests/json_tree.rs | 74 +-- .../encryption/tests/json_tree/generated.rs | 9 +- .../encryption/tests/json_tree_data_plan.rs | 2 +- .../eql/tests/encryption/tests/targets.rs | 42 +- .../sqlx/tests/v3_v4_coexistence_tests.rs | 17 +- packages/stack-encrypt/src/dynamic/record.rs | 4 +- packages/stack-encrypt/src/json/query.rs | 4 +- 46 files changed, 575 insertions(+), 631 deletions(-) diff --git a/languages/golang/encrypt/checker.go b/languages/golang/encrypt/checker.go index 3ea4c81eb..e971b0a2f 100644 --- a/languages/golang/encrypt/checker.go +++ b/languages/golang/encrypt/checker.go @@ -176,7 +176,7 @@ func parseTarget(entry vcvalue.Object) (record.Target, error) { return t, fmt.Errorf("an index is %T, not a string", item) } switch o := record.Output(s); o { - // "json" is the SteVec document index, which no plan output + // "json" is the JSON index a document type carries, which no plan output // carries. case record.Equality, record.Match, record.Ore, record.Ope, record.OreBlock, "json": t.Indexes = append(t.Indexes, o) diff --git a/languages/golang/encrypt/eql/eql_gen.go b/languages/golang/encrypt/eql/eql_gen.go index 9d5b67460..50d9fa6c4 100644 --- a/languages/golang/encrypt/eql/eql_gen.go +++ b/languages/golang/encrypt/eql/eql_gen.go @@ -58,9 +58,8 @@ var Types = []Type{ {Name: "DoubleOrdOre", GoName: "DoubleOrdOre", Family: "double", Suffix: "OrdOre", Plaintext: "float64", SQLDomain: "public.eql_v4_double_ord_ore", Indexes: []string{"ore_block"}, Query: "DoubleOrdOreQuery", QuerySQLDomain: "eql_v4.query_double_ord_ore", Producible: true, Reason: ""}, {Name: "DoubleOrd", GoName: "DoubleOrd", Family: "double", Suffix: "Ord", Plaintext: "float64", SQLDomain: "public.eql_v4_double_ord", Indexes: []string{"ope"}, Query: "DoubleOrdQuery", QuerySQLDomain: "eql_v4.query_double_ord", Producible: true, Reason: ""}, {Name: "DoubleOrdOpe", GoName: "DoubleOrdOpe", Family: "double", Suffix: "OrdOpe", Plaintext: "float64", SQLDomain: "public.eql_v4_double_ord_ope", Indexes: []string{"ope"}, Query: "DoubleOrdOpeQuery", QuerySQLDomain: "eql_v4.query_double_ord_ope", Producible: true, Reason: ""}, - {Name: "SteVecDocument", GoName: "SteVecDocument", Family: "json", Suffix: "Search", Plaintext: "", SQLDomain: "public.eql_v4_json_search", Indexes: []string{"json"}, Query: "SteVecQuery", QuerySQLDomain: "eql_v4.query_json", Producible: false, Reason: "the JSON index is a new operation in the engine (#1060)"}, {Name: "Json", GoName: "JSON", Family: "json", Suffix: "", Plaintext: "", SQLDomain: "public.eql_v4_json", Indexes: nil, Query: "", QuerySQLDomain: "", Producible: false, Reason: "a JSON value seals as a tree of leaves and EQL's `c` holds one; how a document becomes one sealed leaf is not decided"}, - {Name: "EncryptedJson", GoName: "EncryptedJSON", Family: "json_tree", Suffix: "", Plaintext: "json", SQLDomain: "public.eql_v4_json_tree", Indexes: []string{"json"}, Query: "EncryptedJsonQuery", QuerySQLDomain: "eql_v4.query_json_tree", Producible: true, Reason: ""}, + {Name: "EncryptedJson", GoName: "EncryptedJSON", Family: "json_tree", Suffix: "", Plaintext: "json", SQLDomain: "public.eql_v4_json_search", Indexes: []string{"json"}, Query: "EncryptedJsonQuery", QuerySQLDomain: "eql_v4.query_json", Producible: true, Reason: ""}, } // Integer is the EQL type stored in public.eql_v4_integer: the integer family with no index. @@ -2136,53 +2135,6 @@ func (v *DoubleOrdOpeQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// SteVecDocument is the EQL type stored in public.eql_v4_json_search: the json family with the json index. -// The engine cannot produce it yet: the JSON index is a new operation in the engine (#1060). -type SteVecDocument []byte - -// Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. -func (v SteVecDocument) Value() (driver.Value, error) { - return value(v) -} - -// Scan reads a public.eql_v4_json_search column. -func (v *SteVecDocument) Scan(src any) error { - return scan((*[]byte)(v), src, "SteVecDocument") -} - -// MarshalJSON is the EQL value as the JSON it is, or null when empty. -func (v SteVecDocument) MarshalJSON() ([]byte, error) { - return marshal(v, "SteVecDocument") -} - -// UnmarshalJSON reads the EQL value from JSON, as it is. -func (v *SteVecDocument) UnmarshalJSON(b []byte) error { - return unmarshal((*[]byte)(v), b) -} - -// SteVecQuery is the query value for SteVecDocument: the operand eql_v4.query_json takes. -type SteVecQuery []byte - -// Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. -func (v SteVecQuery) Value() (driver.Value, error) { - return value(v) -} - -// Scan reads a eql_v4.query_json column. -func (v *SteVecQuery) Scan(src any) error { - return scan((*[]byte)(v), src, "SteVecQuery") -} - -// MarshalJSON is the EQL value as the JSON it is, or null when empty. -func (v SteVecQuery) MarshalJSON() ([]byte, error) { - return marshal(v, "SteVecQuery") -} - -// UnmarshalJSON reads the EQL value from JSON, as it is. -func (v *SteVecQuery) UnmarshalJSON(b []byte) error { - return unmarshal((*[]byte)(v), b) -} - // JSON is the EQL type stored in public.eql_v4_json: the json family with no index. // The engine cannot produce it yet: a JSON value seals as a tree of leaves and EQL's `c` holds one; how a document becomes one sealed leaf is not decided. type JSON []byte @@ -2207,7 +2159,7 @@ func (v *JSON) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// EncryptedJSON is the EQL type stored in public.eql_v4_json_tree: the json_tree family with the json index. +// EncryptedJSON is the EQL type stored in public.eql_v4_json_search: the json_tree family with the json index. // The engine produces it from a json. type EncryptedJSON []byte @@ -2216,7 +2168,7 @@ func (v EncryptedJSON) Value() (driver.Value, error) { return value(v) } -// Scan reads a public.eql_v4_json_tree column. +// Scan reads a public.eql_v4_json_search column. func (v *EncryptedJSON) Scan(src any) error { return scan((*[]byte)(v), src, "EncryptedJSON") } @@ -2231,7 +2183,7 @@ func (v *EncryptedJSON) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// EncryptedJSONQuery is the query value for EncryptedJSON: the operand eql_v4.query_json_tree takes. +// EncryptedJSONQuery is the query value for EncryptedJSON: the operand eql_v4.query_json takes. type EncryptedJSONQuery []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -2239,7 +2191,7 @@ func (v EncryptedJSONQuery) Value() (driver.Value, error) { return value(v) } -// Scan reads a eql_v4.query_json_tree column. +// Scan reads a eql_v4.query_json column. func (v *EncryptedJSONQuery) Scan(src any) error { return scan((*[]byte)(v), src, "EncryptedJSONQuery") } @@ -2254,7 +2206,7 @@ func (v *EncryptedJSONQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// EncryptedJSONLeafQuery is the json_tree family's ordering operand at a path, compared with an entry `->` extracts: the value eql_v4.query_json_tree_leaf holds. +// EncryptedJSONLeafQuery is the json_tree family's ordering operand at a path, compared with an entry `->` extracts: the value eql_v4.query_json_leaf holds. type EncryptedJSONLeafQuery []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -2262,7 +2214,7 @@ func (v EncryptedJSONLeafQuery) Value() (driver.Value, error) { return value(v) } -// Scan reads a eql_v4.query_json_tree_leaf column. +// Scan reads a eql_v4.query_json_leaf column. func (v *EncryptedJSONLeafQuery) Scan(src any) error { return scan((*[]byte)(v), src, "EncryptedJSONLeafQuery") } @@ -2277,7 +2229,7 @@ func (v *EncryptedJSONLeafQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// EncryptedJSONEntry is the json_tree family's entry: one node of a document, as `->` and the walks return it: the value public.eql_v4_json_tree_entry holds. +// EncryptedJSONEntry is the json_tree family's entry: one node of a document, as `->` and the walks return it: the value public.eql_v4_json_entry holds. type EncryptedJSONEntry []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. @@ -2285,7 +2237,7 @@ func (v EncryptedJSONEntry) Value() (driver.Value, error) { return value(v) } -// Scan reads a public.eql_v4_json_tree_entry column. +// Scan reads a public.eql_v4_json_entry column. func (v *EncryptedJSONEntry) Scan(src any) error { return scan((*[]byte)(v), src, "EncryptedJSONEntry") } diff --git a/packages/eql/crates/eql-bindings/src/encryption/targets.rs b/packages/eql/crates/eql-bindings/src/encryption/targets.rs index 132ed7096..c7af38a13 100644 --- a/packages/eql/crates/eql-bindings/src/encryption/targets.rs +++ b/packages/eql/crates/eql-bindings/src/encryption/targets.rs @@ -47,11 +47,11 @@ //! |---|---| //! | `name` | The type's name, one across Rust, TypeScript and Go: the catalog struct identifier. The data plan's target form names this. (Go spells the `Json` family's types `JSON`; that rename is the Go generator's.) | //! | `family` | The catalog family, lower case: `text`, `integer`, `json`, … | -//! | `suffix` | What a query can do, as the plan's table of suffixes spells it: `""` (stored and read only), `Eq`, `Ord`, `OrdOpe`, `OrdOre`, `Match`, `Search`, `SearchOre`; `Search` for the SteVec document. | +//! | `suffix` | What a query can do, as the plan's table of suffixes spells it: `""` (stored and read only), `Eq`, `Ord`, `OrdOpe`, `OrdOre`, `Match`, `Search`, `SearchOre`; `""` too for a document family's stored type, whose queries are the JSON index's operators. | //! | `plaintext` | The vitaminc [`ValueKind`] name the type is produced from — the same names a plan field's `"type"` key uses — or `null` while the family's plaintext encoding for the stack-encrypt producer profile is unspecified. | //! | `sql_domain` | The schema-qualified PostgreSQL domain the stored value inhabits. | -//! | `indexes` | The indexes the type carries, by the engine's `IndexSpec::key()` names: `eq`, `match`, `ore`, `ope`, and `json` for the SteVec document. A query may ask a field typed with this target for exactly these. | -//! | `query` | The query twin's type name (`TextEqQuery`, `SteVecQuery`), or `null` for a storage-only type, which answers no query. | +//! | `indexes` | The indexes the type carries, by the engine's `IndexSpec::key()` names: `eq`, `match`, `ore`, `ope`, and `json` for a document family's stored type. A query may ask a field typed with this target for exactly these. | +//! | `query` | The query twin's type name (`TextEqQuery`, `EncryptedJsonQuery`), or `null` for a storage-only type, which answers no query. | //! | `query_sql_domain` | The query twin's PostgreSQL domain, `null` likewise. | //! | `producible` | Whether the engine can produce this type today. [`encrypt`] and [`query`] refuse a type that is not, and a generator should too. | //! | `reason` | Why not, when `producible` is `false`; `null` when it is. | diff --git a/packages/eql/crates/eql-bindings/src/v4/json_tree.rs b/packages/eql/crates/eql-bindings/src/v4/json_tree.rs index 9364c45bb..0c29acfd3 100644 --- a/packages/eql/crates/eql-bindings/src/v4/json_tree.rs +++ b/packages/eql/crates/eql-bindings/src/v4/json_tree.rs @@ -74,7 +74,7 @@ pub struct JsonTreeOrderingTerm(pub Base64Text); #[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize, TS, JsonSchema)] pub struct JsonTreeKeyHeader(pub Base64Text); -/// `public.eql_v4_json_tree`: a JSON tree document, `{v, i, h, sv: +/// `public.eql_v4_json_search`: a JSON tree document, `{v, i, h, sv: /// [entry], q: group}`. Strict. `h` is the document's key header, stored /// once (every entry is sealed under the document's one data key); `sv` /// holds one entry per node in depth-first order, the root first; `q` is @@ -93,7 +93,7 @@ pub struct JsonTreeDocument { pub q: JsonTreeQueryGroup, } -/// `public.eql_v4_json_tree_entry`: one node. `s` is the path selector, +/// `public.eql_v4_json_entry`: one node. `s` is the path selector, /// `c` the node's own ciphertext, `x` the self tag by which a read finds /// the node its token names; `p` the parent tag below the chain's heads, /// which a read probes by ordinal; `op` the leaf ordering term. @@ -151,7 +151,7 @@ pub struct JsonTreeQueryGroup { pub el: Vec, } -/// `eql_v4.query_json_tree`: a containment needle, the element-nested +/// `eql_v4.query_json`: a containment needle, the element-nested /// projection of the needle document's selectors, with the alternative /// projection a root scalar carries. Strict. What /// [`stack_encrypt::json::JsonQuery`] serializes to; [`TryFrom`] converts @@ -166,7 +166,7 @@ pub struct JsonTreeQuery { pub alt: Option, } -/// `eql_v4.query_json_tree_leaf`: an ordering operand at a path, the path's +/// `eql_v4.query_json_leaf`: an ordering operand at a path, the path's /// selector and the leaf ordering term. What /// [`stack_encrypt::json::JsonLeafQuery`] serializes to. #[derive(Clone, Debug, PartialEq, Serialize, Deserialize, TS, JsonSchema)] @@ -282,10 +282,10 @@ macro_rules! json_tree_domain_type { }; } -json_tree_domain_type!(JsonTreeDocument, "public.eql_v4_json_tree"); -json_tree_domain_type!(JsonTreeEntry, "public.eql_v4_json_tree_entry"); -json_tree_domain_type!(JsonTreeQuery, "eql_v4.query_json_tree"); -json_tree_domain_type!(JsonTreeLeafQuery, "eql_v4.query_json_tree_leaf"); +json_tree_domain_type!(JsonTreeDocument, "public.eql_v4_json_search"); +json_tree_domain_type!(JsonTreeEntry, "public.eql_v4_json_entry"); +json_tree_domain_type!(JsonTreeQuery, "eql_v4.query_json"); +json_tree_domain_type!(JsonTreeLeafQuery, "eql_v4.query_json_leaf"); // ============================================================================= // Conversions to and from the engine's types, through their one wire form @@ -535,16 +535,16 @@ mod tests { fn the_domains_are_the_v4_json_tree_family() { assert_eq!( JsonTreeDocument::sql_domain_static(), - "public.eql_v4_json_tree" + "public.eql_v4_json_search" ); assert_eq!( JsonTreeEntry::sql_domain_static(), - "public.eql_v4_json_tree_entry" + "public.eql_v4_json_entry" ); - assert_eq!(JsonTreeQuery::sql_domain_static(), "eql_v4.query_json_tree"); + assert_eq!(JsonTreeQuery::sql_domain_static(), "eql_v4.query_json"); assert_eq!( JsonTreeLeafQuery::sql_domain_static(), - "eql_v4.query_json_tree_leaf" + "eql_v4.query_json_leaf" ); } diff --git a/packages/eql/crates/eql-bindings/src/v4/targets.rs b/packages/eql/crates/eql-bindings/src/v4/targets.rs index 793fcffed..abecea153 100644 --- a/packages/eql/crates/eql-bindings/src/v4/targets.rs +++ b/packages/eql/crates/eql-bindings/src/v4/targets.rs @@ -12,9 +12,10 @@ use stack_encrypt::dynamic::ValueFrame; use stack_encrypt::target::IndexSpec; use stack_encrypt::{KeysetCipher, NonEmpty, Pending}; /// Every EQL type a plan may name as a target, in -/// `eql-domains::CATALOG` order: one per stored domain (every flat -/// scalar domain plus the SteVec document). Query twins are not -/// targets; each row names its own under `query`. +/// `eql-domains::CATALOG` order: one per stored scalar domain, then +/// one per document family's stored type +/// (`eql-domains::DOCUMENT_FAMILIES`). Query twins are not targets; +/// each row names its own under `query`. pub const TARGETS: &[Target] = &[ Target { name: "Integer", @@ -604,18 +605,6 @@ pub const TARGETS: &[Target] = &[ producible: true, reason: None, }, - Target { - name: "SteVecDocument", - family: "json", - suffix: "Search", - plaintext: None, - sql_domain: "public.eql_v4_json_search", - indexes: &["json"], - query: Some("SteVecQuery"), - query_sql_domain: Some("eql_v4.query_json"), - producible: false, - reason: Some("the JSON index is a new operation in the engine (#1060)"), - }, Target { name: "Json", family: "json", @@ -635,10 +624,10 @@ pub const TARGETS: &[Target] = &[ family: "json_tree", suffix: "", plaintext: Some("json"), - sql_domain: "public.eql_v4_json_tree", + sql_domain: "public.eql_v4_json_search", indexes: &["json"], query: Some("EncryptedJsonQuery"), - query_sql_domain: Some("eql_v4.query_json_tree"), + query_sql_domain: Some("eql_v4.query_json"), producible: true, reason: None, }, diff --git a/packages/eql/crates/eql-codegen/src/generate.rs b/packages/eql/crates/eql-codegen/src/generate.rs index 30ea4fc2a..df759a566 100644 --- a/packages/eql/crates/eql-codegen/src/generate.rs +++ b/packages/eql/crates/eql-codegen/src/generate.rs @@ -28,6 +28,12 @@ const V3_JSON_FUNCTIONS: &str = "src/v3/json/functions.sql"; /// selecting the leaf is necessarily by name — the same way `dump.rs` resolves /// them. Only the name is spelled here; the SQL type name is DERIVED below. const JSON_ENTRY_DOMAIN: &str = "entry"; +/// The EQL version the json_entry cross surface belongs to. It binds the SteVec +/// entry, whose SQL (`src/v3/json/`) is confined to EQL v3: in v4 the names +/// `json_search`, `json_entry` and `query_json` belong to the JSON tree family +/// (`src/v3/json_tree/`), so a cross file rendered as v4 would bind to a type +/// with another wire shape, or fail its `-- REQUIRE:` on a dropped file. +const JSON_ENTRY_VERSION: crate::render::EqlVersion = crate::render::EqlVersion::V3; /// The fixed cross type: the extracted SteVec leaf, `public.eql_v3_json_entry`. /// @@ -54,6 +60,20 @@ fn json_entry_stem(family_name: &str) -> String { format!("json_entry_{family_name}") } +/// Confine a rendered json_entry cross file to [`JSON_ENTRY_VERSION`]: its +/// `-- ONLY EQL VERSION:` line, after the generated-file marker, as the range +/// files carry theirs. +fn confine_json_entry_file(body: String) -> String { + let (marker, rest) = body + .split_once('\n') + .expect("a rendered file starts with its generated-file marker line"); + format!( + "{marker}\n{} {}\n{rest}", + crate::render::ONLY_DIRECTIVE, + JSON_ENTRY_VERSION.number() + ) +} + /// REQUIRE path for a family's generated json_entry cross functions file. fn json_entry_functions_path(family_name: &str) -> String { scalar_path( @@ -707,11 +727,13 @@ pub fn render_json_entry_cross_functions( dom: json_entry, entries, }; - environment() - .get_template("functions.sql") - .unwrap() - .render(&ctx) - .expect("render json_entry cross functions") + confine_json_entry_file( + environment() + .get_template("functions.sql") + .unwrap() + .render(&ctx) + .expect("render json_entry cross functions"), + ) } /// Body for a family's json_entry__operators.sql: a CREATE @@ -779,11 +801,13 @@ pub fn render_json_entry_cross_operators( dom: json_entry, operators, }; - environment() - .get_template("operators.sql") - .unwrap() - .render(&ctx) - .expect("render json_entry cross operators") + confine_json_entry_file( + environment() + .get_template("operators.sql") + .unwrap() + .render(&ctx) + .expect("render json_entry cross operators"), + ) } /// Body for a domain's _aggregates.sql, or None if not ord-capable. @@ -994,7 +1018,8 @@ pub fn render_type(spec: &DomainFamily, out_dir: &Path) -> Vec<(PathBuf, String) // (date/timestamp — no native JSON leaf; text's Bloom-bearing `search`) get // blocker-only files so the pair raises instead of flattening to native // `jsonb jsonb`. Both lists empty for boolean (storage-only) and json - // (no scalar operands) → no cross files. + // (no scalar operands) → no cross files. Both files carry + // `-- ONLY EQL VERSION: 3`, as the SteVec SQL they bind does. let cross_domains = json_entry_cross_domains(spec); let cross_blocked = json_entry_cross_blocked_domains(spec); if !(cross_domains.is_empty() && cross_blocked.is_empty()) { diff --git a/packages/eql/crates/eql-codegen/src/go_eql.rs b/packages/eql/crates/eql-codegen/src/go_eql.rs index c6b3608ad..01ddeb29b 100644 --- a/packages/eql/crates/eql-codegen/src/go_eql.rs +++ b/packages/eql/crates/eql-codegen/src/go_eql.rs @@ -280,8 +280,8 @@ mod tests { } assert!(out.contains( "Name: \"EncryptedJson\", GoName: \"EncryptedJSON\", Family: \"json_tree\", Suffix: \"\", \ - Plaintext: \"json\", SQLDomain: \"public.eql_v4_json_tree\", Indexes: []string{\"json\"}, \ - Query: \"EncryptedJsonQuery\", QuerySQLDomain: \"eql_v4.query_json_tree\", Producible: true" + Plaintext: \"json\", SQLDomain: \"public.eql_v4_json_search\", Indexes: []string{\"json\"}, \ + Query: \"EncryptedJsonQuery\", QuerySQLDomain: \"eql_v4.query_json\", Producible: true" )); } @@ -310,11 +310,10 @@ mod tests { ); } } - // The one query type two rows share (text_ord and text_ord_ope - // have distinct names; the SteVec needle belongs to one document), - // so no type is declared twice. + // No type is declared twice, and the SteVec document, EQL v3 only, + // has no v4 type. assert_eq!(out.matches("\ntype JSON []byte\n").count(), 1); - assert_eq!(out.matches("\ntype SteVecQuery []byte\n").count(), 1); + assert!(!out.contains("SteVec")); // TextEq is producible and says so; Json is not and says why. assert!(out.contains("// TextEq is the EQL type stored in public.eql_v4_text_eq")); assert!(out.contains("Producible: true, Reason: \"\"")); diff --git a/packages/eql/crates/eql-codegen/src/targets.rs b/packages/eql/crates/eql-codegen/src/targets.rs index aa8670fd7..c42b532aa 100644 --- a/packages/eql/crates/eql-codegen/src/targets.rs +++ b/packages/eql/crates/eql-codegen/src/targets.rs @@ -69,11 +69,6 @@ pub fn index_key(term: Term) -> &'static str { } } -/// The index a SteVec document carries: the engine's JSON index, which has -/// no catalog `Term` because its terms live per `sv` leaf, not as flat -/// payload keys. -const JSON_INDEX_KEY: &str = "json"; - /// An operator a producible EQL type answers in Stack Encrypt's typed query /// chain: what the generated file implements `op::Answers` for. #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -194,10 +189,18 @@ pub(crate) struct Row { pub(crate) document: bool, } -/// Every row, in catalog order: one per stored domain, then one per -/// document family's stored domain. +/// The catalog domains that are targets, in catalog order: every stored +/// scalar domain. The `json` family's SteVec document is not one: its SQL is +/// EQL v3 only, and in v4 its domain name, `json_search`, is the JSON tree +/// family's stored type, a document family's row. +fn target_domains() -> impl Iterator { + stored_payload_domains().filter(|(_, d)| d.is_scalar()) +} + +/// Every row, in catalog order: one per [target domain](target_domains), +/// then one per document family's stored domain. pub(crate) fn rows() -> Vec { - stored_payload_domains() + target_domains() .map(|(f, d)| row(f, d)) .chain(eql_domains::DOCUMENT_FAMILIES.iter().map(document_row)) .collect() @@ -243,36 +246,11 @@ fn suffix(domain: &Domain) -> String { } fn row(family: &'static DomainFamily, domain: &'static Domain) -> Row { - let is_stevec = matches!(domain.shape, Shape::SteVec); - let indexes = if is_stevec { - vec![JSON_INDEX_KEY] - } else { - Term::payload_terms(domain.terms) - .into_iter() - .map(index_key) - .collect() - }; - let query = if is_stevec { - // The containment needle is the document's query form. Its struct - // is the hand-written `SteVecQuery`; its domain is the family's - // `query` domain (`eql_v4.query_json`), read from the catalog so - // the name is spelled once. v4 has no compiled SteVec types yet, so - // these rows are never producible. - family - .domains - .iter() - .find(|d| matches!(d.shape, Shape::SteVec) && d.name == "query") - .map(|needle| { - ( - needle.rust_struct_name(family.name), - format!( - "{}.{}", - TARGET_VERSION.prefix(), - needle.full_name(family.name) - ), - ) - }) - } else if domain.terms.is_empty() { + let indexes: Vec<&'static str> = Term::payload_terms(domain.terms) + .into_iter() + .map(index_key) + .collect(); + let query = if domain.terms.is_empty() { None } else { Some(( @@ -303,7 +281,7 @@ fn row(family: &'static DomainFamily, domain: &'static Domain) -> Row { ), indexes, query, - projections: if !is_stevec && v4_optional_query_terms(domain) { + projections: if v4_optional_query_terms(domain) { Term::payload_terms(domain.terms) .into_iter() .map(|term| { @@ -580,9 +558,10 @@ fn render_targets_from(rows: &[Row]) -> String { use stack_encrypt::{KeysetCipher, NonEmpty, Pending}; /// Every EQL type a plan may name as a target, in - /// `eql-domains::CATALOG` order: one per stored domain (every flat - /// scalar domain plus the SteVec document). Query twins are not - /// targets; each row names its own under `query`. + /// `eql-domains::CATALOG` order: one per stored scalar domain, then + /// one per document family's stored type + /// (`eql-domains::DOCUMENT_FAMILIES`). Query twins are not targets; + /// each row names its own under `query`. pub const TARGETS: &[Target] = &[ #entries ]; @@ -850,20 +829,21 @@ mod tests { // whitespace-collapsed line so the assertions below do not depend on // where it broke. let out: String = rendered.split_whitespace().collect::>().join(" "); - // One row per stored domain, in catalog order, named as the struct. + // One row per target domain, in catalog order, named as the struct. let mut last = 0; - for (family, domain) in stored_payload_domains() { + for (family, domain) in target_domains() { let needle = format!("name: \"{}\",", domain.rust_struct_name(family.name)); let at = out[last..] .find(&needle) .unwrap_or_else(|| panic!("{needle} missing or out of order")); last += at + needle.len(); } - // The SteVec document carries the JSON index and the containment - // needle as its query form; the scalar storage domain carries none. - assert!(out.contains("name: \"SteVecDocument\"")); + // The SteVec document is EQL v3 only, so it is no target: the + // v4 `json_search` domain is the JSON tree family's stored type, + // `JsonSearch`, which carries the JSON index and the containment + // needle as its query form. + assert!(!out.contains("SteVec")); assert!(out.contains("indexes: &[\"json\"]")); - assert!(out.contains("query: Some(\"SteVecQuery\")")); assert!(out.contains("query_sql_domain: Some(\"eql_v4.query_json\")")); assert!(out.contains("sql_domain: \"public.eql_v4_text_eq\"")); assert!(out.contains("query_sql_domain: Some(\"eql_v4.query_text_eq\")")); diff --git a/packages/eql/crates/eql-domains/src/lib.rs b/packages/eql/crates/eql-domains/src/lib.rs index 49d21cd9c..8baafcb2f 100644 --- a/packages/eql/crates/eql-domains/src/lib.rs +++ b/packages/eql/crates/eql-domains/src/lib.rs @@ -618,7 +618,7 @@ impl DocumentFamily { /// with exact containment, equality and ordering at paths, `->` extraction /// and subtree walks (`docs/plans/2026-10-17-json-tree-index.md`). The /// stored type is named for its role, `EncryptedJson`, over the SQL domain -/// `public.eql_v4_json_tree`; the Rust struct keeps its family name. +/// `public.eql_v4_json_search`; the Rust struct keeps its family name. pub const JSON_TREE: DocumentFamily = DocumentFamily { name: "json_tree", plaintext: "json", @@ -628,25 +628,25 @@ pub const JSON_TREE: DocumentFamily = DocumentFamily { role: DocumentRole::Stored, name: "EncryptedJson", rust: "JsonTreeDocument", - sql_domain: "public.eql_v4_json_tree", + sql_domain: "public.eql_v4_json_search", }, DocumentDomain { role: DocumentRole::Query, name: "EncryptedJsonQuery", rust: "JsonTreeQuery", - sql_domain: "eql_v4.query_json_tree", + sql_domain: "eql_v4.query_json", }, DocumentDomain { role: DocumentRole::LeafQuery, name: "EncryptedJsonLeafQuery", rust: "JsonTreeLeafQuery", - sql_domain: "eql_v4.query_json_tree_leaf", + sql_domain: "eql_v4.query_json_leaf", }, DocumentDomain { role: DocumentRole::Entry, name: "EncryptedJsonEntry", rust: "JsonTreeEntry", - sql_domain: "public.eql_v4_json_tree_entry", + sql_domain: "public.eql_v4_json_entry", }, ], }; diff --git a/packages/eql/crates/eql-domains/src/tests.rs b/packages/eql/crates/eql-domains/src/tests.rs index df0de542e..00d4d42a7 100644 --- a/packages/eql/crates/eql-domains/src/tests.rs +++ b/packages/eql/crates/eql-domains/src/tests.rs @@ -1666,7 +1666,7 @@ mod document_families { fn json_tree_is_the_encrypted_json_target() { let stored = JSON_TREE.domain(DocumentRole::Stored).unwrap(); assert_eq!(stored.name, "EncryptedJson"); - assert_eq!(stored.sql_domain, "public.eql_v4_json_tree"); + assert_eq!(stored.sql_domain, "public.eql_v4_json_search"); assert_eq!(JSON_TREE.plaintext, "json"); assert_eq!(JSON_TREE.index, "json"); } diff --git a/packages/eql/docs/README.md b/packages/eql/docs/README.md index b5b9d7e84..ac896c2ea 100644 --- a/packages/eql/docs/README.md +++ b/packages/eql/docs/README.md @@ -14,7 +14,7 @@ This directory contains the documentation for the Encrypt Query Language (EQL). - [Writing fast queries against EQL columns](reference/query-performance.md) - Performance overview (points to Database Indexes) - [Adding a Scalar Encrypted-Domain Type](reference/adding-a-scalar-encrypted-domain-type.md) - How the `eql_v3.` domain families are generated - [EQL with JSON and JSONB](reference/json-support.md) -- [What an encrypted JSON tree reveals](reference/json-tree-leakage.md) - What `public.eql_v4_json_tree` (EQL v4) shows the database at rest and at query time, and what it does not +- [What an encrypted JSON tree reveals](reference/json-tree-leakage.md) - What `public.eql_v4_json_search` (EQL v4) shows the database at rest and at query time, and what it does not - [EQL payload / wire format](../crates/eql-bindings/README.md) - Canonical wire types for the encrypted payload (envelope `v`/`i`/`c` and the `hm`/`op`/`ob`/`bf` index terms) - [Client-side index configuration](https://cipherstash.com/docs/stack/cipherstash/encryption/schema) - Configuring searchable encryption in CipherStash Stack / CipherStash Proxy diff --git a/packages/eql/docs/reference/json-tree-leakage.md b/packages/eql/docs/reference/json-tree-leakage.md index a5130d0e5..c1f2f60fe 100644 --- a/packages/eql/docs/reference/json-tree-leakage.md +++ b/packages/eql/docs/reference/json-tree-leakage.md @@ -1,6 +1,6 @@ # What an encrypted JSON tree reveals -`public.eql_v4_json_tree` (EQL v4) stores a JSON document so that PostgreSQL can answer containment, path extraction, ordering at chosen paths and subtree reads without the key. Every value, key name and container sits inside an AES-256-GCM ciphertext under the document's data key, and nothing on this page shows the text of one. What the database can see is **structure**, and a great deal of it: which paths a row has and how many times, the order the nodes are stored in, which (path, value) pairs repeat across rows, and lengths. Each item below is what one query capability or one storage choice costs. Read this before you put a column on it, and decide the two options at the end with it in mind. +`public.eql_v4_json_search` (EQL v4) stores a JSON document so that PostgreSQL can answer containment, path extraction, ordering at chosen paths and subtree reads without the key. Every value, key name and container sits inside an AES-256-GCM ciphertext under the document's data key, and nothing on this page shows the text of one. What the database can see is **structure**, and a great deal of it: which paths a row has and how many times, the order the nodes are stored in, which (path, value) pairs repeat across rows, and lengths. Each item below is what one query capability or one storage choice costs. Read this before you put a column on it, and decide the two options at the end with it in mind. ## On this page diff --git a/packages/eql/src/v3/json/aggregates.sql b/packages/eql/src/v3/json/aggregates.sql index 50621e353..9503720d6 100644 --- a/packages/eql/src/v3/json/aggregates.sql +++ b/packages/eql/src/v3/json/aggregates.sql @@ -1,3 +1,4 @@ +-- ONLY EQL VERSION: 3 -- REQUIRE: src/v3/json/types.sql -- REQUIRE: src/v3/json/functions.sql -- REQUIRE: src/v3/sem/ope_cllw/types.sql diff --git a/packages/eql/src/v3/json/blockers.sql b/packages/eql/src/v3/json/blockers.sql index cd9ee7b01..583568d4e 100644 --- a/packages/eql/src/v3/json/blockers.sql +++ b/packages/eql/src/v3/json/blockers.sql @@ -1,3 +1,4 @@ +-- ONLY EQL VERSION: 3 -- REQUIRE: src/v3/schema.sql -- REQUIRE: src/v3/json/types.sql -- REQUIRE: src/v3/scalars/functions.sql diff --git a/packages/eql/src/v3/json/functions.sql b/packages/eql/src/v3/json/functions.sql index d8ba0e59c..b5f00798d 100644 --- a/packages/eql/src/v3/json/functions.sql +++ b/packages/eql/src/v3/json/functions.sql @@ -1,3 +1,4 @@ +-- ONLY EQL VERSION: 3 -- REQUIRE: src/v3/schema.sql -- REQUIRE: src/v3/json/types.sql -- REQUIRE: src/v3/sem/ope_cllw/types.sql diff --git a/packages/eql/src/v3/json/operators.sql b/packages/eql/src/v3/json/operators.sql index e520f47f3..fbdeb44aa 100644 --- a/packages/eql/src/v3/json/operators.sql +++ b/packages/eql/src/v3/json/operators.sql @@ -1,3 +1,4 @@ +-- ONLY EQL VERSION: 3 -- REQUIRE: src/v3/schema.sql -- REQUIRE: src/v3/json/types.sql -- REQUIRE: src/v3/json/functions.sql diff --git a/packages/eql/src/v3/json/types.sql b/packages/eql/src/v3/json/types.sql index 3a676fa48..3daca3e50 100644 --- a/packages/eql/src/v3/json/types.sql +++ b/packages/eql/src/v3/json/types.sql @@ -1,3 +1,4 @@ +-- ONLY EQL VERSION: 3 -- REQUIRE: src/v3/schema.sql --! @file v3/json/types.sql diff --git a/packages/eql/src/v3/json_tree/blockers.sql b/packages/eql/src/v3/json_tree/blockers.sql index bff60387e..57615112a 100644 --- a/packages/eql/src/v3/json_tree/blockers.sql +++ b/packages/eql/src/v3/json_tree/blockers.sql @@ -4,15 +4,15 @@ -- REQUIRE: src/v3/scalars/functions.sql --! @file v3/json_tree/blockers.sql ---! @brief Native-jsonb firewall for public.{{prefix}}_json_tree. +--! @brief Native-jsonb firewall for public.{{prefix}}_json_search. --! ---! public.{{prefix}}_json_tree SUPPORTS @> <@ -> ->> (see operators.sql), and ---! public.{{prefix}}_json_tree_entry the ordered comparisons < <= > >= against an +--! public.{{prefix}}_json_search SUPPORTS @> <@ -> ->> (see operators.sql), and +--! public.{{prefix}}_json_entry the ordered comparisons < <= > >= against an --! ordering operand or another entry. Entry equality = <> is blocked: an --! entry's ordering term is not an exact equality representation, and exact --! equality at a path is containment on a value selector. An entry against --! plain jsonb is blocked for every comparison, in both directions, so an ---! ordering operand that lost its `{{prefix}}.query_json_tree_leaf` cast +--! ordering operand that lost its `{{prefix}}.query_json_leaf` cast --! raises instead of comparing ciphertext. Root-document comparisons are --! blocked too. `ORDER BY col -> 'sel'` cannot be blocked without an operator --! class, which needs a superuser: it sorts by jsonb's ordering of the entry, @@ -28,1091 +28,1091 @@ --! supported', rather than failing earlier on a boolean intermediate. --! @brief Blocker: ? (key/element exists). ---! @param a public.{{prefix}}_json_tree Left operand (encrypted payload). +--! @param a public.{{prefix}}_json_search Left operand (encrypted payload). --! @param b text Native RHS operand. --! @return boolean Never returns; always raises 'operator not supported'. -CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_exists(a public.{{prefix}}_json_tree, b text) +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_exists(a public.{{prefix}}_json_search, b text) RETURNS boolean IMMUTABLE PARALLEL SAFE SET search_path = pg_catalog, extensions, public AS $$ BEGIN - RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree', '?'); + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_search', '?'); END; $$ LANGUAGE plpgsql; CREATE OPERATOR ? ( FUNCTION = {{prefix}}_internal.json_tree_blocked_exists, - LEFTARG = public.{{prefix}}_json_tree, + LEFTARG = public.{{prefix}}_json_search, RIGHTARG = text ); --! @brief Blocker: ?| (any key exists). ---! @param a public.{{prefix}}_json_tree Left operand (encrypted payload). +--! @param a public.{{prefix}}_json_search Left operand (encrypted payload). --! @param b text[] Native RHS operand. --! @return boolean Never returns; always raises 'operator not supported'. -CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_exists_any(a public.{{prefix}}_json_tree, b text[]) +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_exists_any(a public.{{prefix}}_json_search, b text[]) RETURNS boolean IMMUTABLE PARALLEL SAFE SET search_path = pg_catalog, extensions, public AS $$ BEGIN - RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree', '?|'); + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_search', '?|'); END; $$ LANGUAGE plpgsql; CREATE OPERATOR ?| ( FUNCTION = {{prefix}}_internal.json_tree_blocked_exists_any, - LEFTARG = public.{{prefix}}_json_tree, + LEFTARG = public.{{prefix}}_json_search, RIGHTARG = text[] ); --! @brief Blocker: ?& (all keys exist). ---! @param a public.{{prefix}}_json_tree Left operand (encrypted payload). +--! @param a public.{{prefix}}_json_search Left operand (encrypted payload). --! @param b text[] Native RHS operand. --! @return boolean Never returns; always raises 'operator not supported'. -CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_exists_all(a public.{{prefix}}_json_tree, b text[]) +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_exists_all(a public.{{prefix}}_json_search, b text[]) RETURNS boolean IMMUTABLE PARALLEL SAFE SET search_path = pg_catalog, extensions, public AS $$ BEGIN - RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree', '?&'); + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_search', '?&'); END; $$ LANGUAGE plpgsql; CREATE OPERATOR ?& ( FUNCTION = {{prefix}}_internal.json_tree_blocked_exists_all, - LEFTARG = public.{{prefix}}_json_tree, + LEFTARG = public.{{prefix}}_json_search, RIGHTARG = text[] ); --! @brief Blocker: @? (jsonpath exists). ---! @param a public.{{prefix}}_json_tree Left operand (encrypted payload). +--! @param a public.{{prefix}}_json_search Left operand (encrypted payload). --! @param b jsonpath Native RHS operand. --! @return boolean Never returns; always raises 'operator not supported'. -CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_jsonpath_exists(a public.{{prefix}}_json_tree, b jsonpath) +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_jsonpath_exists(a public.{{prefix}}_json_search, b jsonpath) RETURNS boolean IMMUTABLE PARALLEL SAFE SET search_path = pg_catalog, extensions, public AS $$ BEGIN - RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree', '@?'); + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_search', '@?'); END; $$ LANGUAGE plpgsql; CREATE OPERATOR @? ( FUNCTION = {{prefix}}_internal.json_tree_blocked_jsonpath_exists, - LEFTARG = public.{{prefix}}_json_tree, + LEFTARG = public.{{prefix}}_json_search, RIGHTARG = jsonpath ); --! @brief Blocker: @@ (jsonpath predicate). ---! @param a public.{{prefix}}_json_tree Left operand (encrypted payload). +--! @param a public.{{prefix}}_json_search Left operand (encrypted payload). --! @param b jsonpath Native RHS operand. --! @return boolean Never returns; always raises 'operator not supported'. -CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_jsonpath_match(a public.{{prefix}}_json_tree, b jsonpath) +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_jsonpath_match(a public.{{prefix}}_json_search, b jsonpath) RETURNS boolean IMMUTABLE PARALLEL SAFE SET search_path = pg_catalog, extensions, public AS $$ BEGIN - RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree', '@@'); + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_search', '@@'); END; $$ LANGUAGE plpgsql; CREATE OPERATOR @@ ( FUNCTION = {{prefix}}_internal.json_tree_blocked_jsonpath_match, - LEFTARG = public.{{prefix}}_json_tree, + LEFTARG = public.{{prefix}}_json_search, RIGHTARG = jsonpath ); --! @brief Blocker: #> (path extract, native returns jsonb). ---! @param a public.{{prefix}}_json_tree Left operand (encrypted payload). +--! @param a public.{{prefix}}_json_search Left operand (encrypted payload). --! @param b text[] Native RHS operand. --! @return jsonb Never returns; always raises 'operator not supported'. -CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_path_extract(a public.{{prefix}}_json_tree, b text[]) +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_path_extract(a public.{{prefix}}_json_search, b text[]) RETURNS jsonb IMMUTABLE PARALLEL SAFE SET search_path = pg_catalog, extensions, public AS $$ BEGIN - RETURN {{prefix}}_internal.encrypted_domain_unsupported_jsonb('public.{{prefix}}_json_tree', '#>'); + RETURN {{prefix}}_internal.encrypted_domain_unsupported_jsonb('public.{{prefix}}_json_search', '#>'); END; $$ LANGUAGE plpgsql; CREATE OPERATOR #> ( FUNCTION = {{prefix}}_internal.json_tree_blocked_path_extract, - LEFTARG = public.{{prefix}}_json_tree, + LEFTARG = public.{{prefix}}_json_search, RIGHTARG = text[] ); --! @brief Blocker: #>> (path extract as text). ---! @param a public.{{prefix}}_json_tree Left operand (encrypted payload). +--! @param a public.{{prefix}}_json_search Left operand (encrypted payload). --! @param b text[] Native RHS operand. --! @return text Never returns; always raises 'operator not supported'. -CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_path_extract_text(a public.{{prefix}}_json_tree, b text[]) +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_path_extract_text(a public.{{prefix}}_json_search, b text[]) RETURNS text IMMUTABLE PARALLEL SAFE SET search_path = pg_catalog, extensions, public AS $$ BEGIN - RETURN {{prefix}}_internal.encrypted_domain_unsupported_text('public.{{prefix}}_json_tree', '#>>'); + RETURN {{prefix}}_internal.encrypted_domain_unsupported_text('public.{{prefix}}_json_search', '#>>'); END; $$ LANGUAGE plpgsql; CREATE OPERATOR #>> ( FUNCTION = {{prefix}}_internal.json_tree_blocked_path_extract_text, - LEFTARG = public.{{prefix}}_json_tree, + LEFTARG = public.{{prefix}}_json_search, RIGHTARG = text[] ); --! @brief Blocker: - (delete key, text RHS). ---! @param a public.{{prefix}}_json_tree Left operand (encrypted payload). +--! @param a public.{{prefix}}_json_search Left operand (encrypted payload). --! @param b text Native RHS operand. --! @return jsonb Never returns; always raises 'operator not supported'. -CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_delete_text(a public.{{prefix}}_json_tree, b text) +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_delete_text(a public.{{prefix}}_json_search, b text) RETURNS jsonb IMMUTABLE PARALLEL SAFE SET search_path = pg_catalog, extensions, public AS $$ BEGIN - RETURN {{prefix}}_internal.encrypted_domain_unsupported_jsonb('public.{{prefix}}_json_tree', '-'); + RETURN {{prefix}}_internal.encrypted_domain_unsupported_jsonb('public.{{prefix}}_json_search', '-'); END; $$ LANGUAGE plpgsql; CREATE OPERATOR - ( FUNCTION = {{prefix}}_internal.json_tree_blocked_delete_text, - LEFTARG = public.{{prefix}}_json_tree, + LEFTARG = public.{{prefix}}_json_search, RIGHTARG = text ); --! @brief Blocker: - (delete index, integer RHS). ---! @param a public.{{prefix}}_json_tree Left operand (encrypted payload). +--! @param a public.{{prefix}}_json_search Left operand (encrypted payload). --! @param b integer Native RHS operand. --! @return jsonb Never returns; always raises 'operator not supported'. -CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_delete_int(a public.{{prefix}}_json_tree, b integer) +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_delete_int(a public.{{prefix}}_json_search, b integer) RETURNS jsonb IMMUTABLE PARALLEL SAFE SET search_path = pg_catalog, extensions, public AS $$ BEGIN - RETURN {{prefix}}_internal.encrypted_domain_unsupported_jsonb('public.{{prefix}}_json_tree', '-'); + RETURN {{prefix}}_internal.encrypted_domain_unsupported_jsonb('public.{{prefix}}_json_search', '-'); END; $$ LANGUAGE plpgsql; CREATE OPERATOR - ( FUNCTION = {{prefix}}_internal.json_tree_blocked_delete_int, - LEFTARG = public.{{prefix}}_json_tree, + LEFTARG = public.{{prefix}}_json_search, RIGHTARG = integer ); --! @brief Blocker: - (delete keys, text[] RHS). ---! @param a public.{{prefix}}_json_tree Left operand (encrypted payload). +--! @param a public.{{prefix}}_json_search Left operand (encrypted payload). --! @param b text[] Native RHS operand. --! @return jsonb Never returns; always raises 'operator not supported'. -CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_delete_array(a public.{{prefix}}_json_tree, b text[]) +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_delete_array(a public.{{prefix}}_json_search, b text[]) RETURNS jsonb IMMUTABLE PARALLEL SAFE SET search_path = pg_catalog, extensions, public AS $$ BEGIN - RETURN {{prefix}}_internal.encrypted_domain_unsupported_jsonb('public.{{prefix}}_json_tree', '-'); + RETURN {{prefix}}_internal.encrypted_domain_unsupported_jsonb('public.{{prefix}}_json_search', '-'); END; $$ LANGUAGE plpgsql; CREATE OPERATOR - ( FUNCTION = {{prefix}}_internal.json_tree_blocked_delete_array, - LEFTARG = public.{{prefix}}_json_tree, + LEFTARG = public.{{prefix}}_json_search, RIGHTARG = text[] ); --! @brief Blocker: #- (delete at path). ---! @param a public.{{prefix}}_json_tree Left operand (encrypted payload). +--! @param a public.{{prefix}}_json_search Left operand (encrypted payload). --! @param b text[] Native RHS operand. --! @return jsonb Never returns; always raises 'operator not supported'. -CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_delete_path(a public.{{prefix}}_json_tree, b text[]) +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_delete_path(a public.{{prefix}}_json_search, b text[]) RETURNS jsonb IMMUTABLE PARALLEL SAFE SET search_path = pg_catalog, extensions, public AS $$ BEGIN - RETURN {{prefix}}_internal.encrypted_domain_unsupported_jsonb('public.{{prefix}}_json_tree', '#-'); + RETURN {{prefix}}_internal.encrypted_domain_unsupported_jsonb('public.{{prefix}}_json_search', '#-'); END; $$ LANGUAGE plpgsql; CREATE OPERATOR #- ( FUNCTION = {{prefix}}_internal.json_tree_blocked_delete_path, - LEFTARG = public.{{prefix}}_json_tree, + LEFTARG = public.{{prefix}}_json_search, RIGHTARG = text[] ); --! @brief Blocker: || (concatenate, encrypted on the left). ---! @param a public.{{prefix}}_json_tree Left operand (encrypted payload). +--! @param a public.{{prefix}}_json_search Left operand (encrypted payload). --! @param b jsonb Native RHS operand. --! @return jsonb Never returns; always raises 'operator not supported'. -CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_concat(a public.{{prefix}}_json_tree, b jsonb) +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_concat(a public.{{prefix}}_json_search, b jsonb) RETURNS jsonb IMMUTABLE PARALLEL SAFE SET search_path = pg_catalog, extensions, public AS $$ BEGIN - RETURN {{prefix}}_internal.encrypted_domain_unsupported_jsonb('public.{{prefix}}_json_tree', '||'); + RETURN {{prefix}}_internal.encrypted_domain_unsupported_jsonb('public.{{prefix}}_json_search', '||'); END; $$ LANGUAGE plpgsql; CREATE OPERATOR || ( FUNCTION = {{prefix}}_internal.json_tree_blocked_concat, - LEFTARG = public.{{prefix}}_json_tree, + LEFTARG = public.{{prefix}}_json_search, RIGHTARG = jsonb ); --! @brief Blocker: || (concatenate, encrypted on the right). --! @param a jsonb Native LHS operand. ---! @param b public.{{prefix}}_json_tree Right operand (encrypted payload). +--! @param b public.{{prefix}}_json_search Right operand (encrypted payload). --! @return jsonb Never returns; always raises 'operator not supported'. -CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_concat_rhs(a jsonb, b public.{{prefix}}_json_tree) +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_concat_rhs(a jsonb, b public.{{prefix}}_json_search) RETURNS jsonb IMMUTABLE PARALLEL SAFE SET search_path = pg_catalog, extensions, public AS $$ BEGIN - RETURN {{prefix}}_internal.encrypted_domain_unsupported_jsonb('public.{{prefix}}_json_tree', '||'); + RETURN {{prefix}}_internal.encrypted_domain_unsupported_jsonb('public.{{prefix}}_json_search', '||'); END; $$ LANGUAGE plpgsql; CREATE OPERATOR || ( FUNCTION = {{prefix}}_internal.json_tree_blocked_concat_rhs, LEFTARG = jsonb, - RIGHTARG = public.{{prefix}}_json_tree + RIGHTARG = public.{{prefix}}_json_search ); --! @brief Blocker: -> (element by position): positions are tokens, not selectors. ---! @param a public.{{prefix}}_json_tree Left operand (encrypted payload). +--! @param a public.{{prefix}}_json_search Left operand (encrypted payload). --! @param b integer Native RHS operand. --! @return jsonb Never returns; always raises 'operator not supported'. -CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_select_int(a public.{{prefix}}_json_tree, b integer) +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_select_int(a public.{{prefix}}_json_search, b integer) RETURNS jsonb IMMUTABLE PARALLEL SAFE SET search_path = pg_catalog, extensions, public AS $$ BEGIN - RETURN {{prefix}}_internal.encrypted_domain_unsupported_jsonb('public.{{prefix}}_json_tree', '->'); + RETURN {{prefix}}_internal.encrypted_domain_unsupported_jsonb('public.{{prefix}}_json_search', '->'); END; $$ LANGUAGE plpgsql; CREATE OPERATOR -> ( FUNCTION = {{prefix}}_internal.json_tree_blocked_select_int, - LEFTARG = public.{{prefix}}_json_tree, + LEFTARG = public.{{prefix}}_json_search, RIGHTARG = integer ); --! @brief Blocker: ->> (element by position, as text). ---! @param a public.{{prefix}}_json_tree Left operand (encrypted payload). +--! @param a public.{{prefix}}_json_search Left operand (encrypted payload). --! @param b integer Native RHS operand. --! @return text Never returns; always raises 'operator not supported'. -CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_select_int_text(a public.{{prefix}}_json_tree, b integer) +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_select_int_text(a public.{{prefix}}_json_search, b integer) RETURNS text IMMUTABLE PARALLEL SAFE SET search_path = pg_catalog, extensions, public AS $$ BEGIN - RETURN {{prefix}}_internal.encrypted_domain_unsupported_text('public.{{prefix}}_json_tree', '->>'); + RETURN {{prefix}}_internal.encrypted_domain_unsupported_text('public.{{prefix}}_json_search', '->>'); END; $$ LANGUAGE plpgsql; CREATE OPERATOR ->> ( FUNCTION = {{prefix}}_internal.json_tree_blocked_select_int_text, - LEFTARG = public.{{prefix}}_json_tree, + LEFTARG = public.{{prefix}}_json_search, RIGHTARG = integer ); --! @brief Blocker: root document = (document, document). ---! @param a public.{{prefix}}_json_tree Left operand (encrypted payload). ---! @param b public.{{prefix}}_json_tree Right operand (encrypted payload). +--! @param a public.{{prefix}}_json_search Left operand (encrypted payload). +--! @param b public.{{prefix}}_json_search Right operand (encrypted payload). --! @return boolean Never returns; always raises 'operator not supported'. -CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_compare_eq_doc_doc(a public.{{prefix}}_json_tree, b public.{{prefix}}_json_tree) +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_compare_eq_doc_doc(a public.{{prefix}}_json_search, b public.{{prefix}}_json_search) RETURNS boolean IMMUTABLE PARALLEL SAFE SET search_path = pg_catalog, extensions, public AS $$ BEGIN - RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree', '='); + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_search', '='); END; $$ LANGUAGE plpgsql; CREATE OPERATOR = ( FUNCTION = {{prefix}}_internal.json_tree_blocked_compare_eq_doc_doc, - LEFTARG = public.{{prefix}}_json_tree, - RIGHTARG = public.{{prefix}}_json_tree + LEFTARG = public.{{prefix}}_json_search, + RIGHTARG = public.{{prefix}}_json_search ); --! @brief Blocker: root document = (document, jsonb). ---! @param a public.{{prefix}}_json_tree Left operand (encrypted payload). +--! @param a public.{{prefix}}_json_search Left operand (encrypted payload). --! @param b jsonb Native RHS operand. --! @return boolean Never returns; always raises 'operator not supported'. -CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_compare_eq_doc_jsonb(a public.{{prefix}}_json_tree, b jsonb) +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_compare_eq_doc_jsonb(a public.{{prefix}}_json_search, b jsonb) RETURNS boolean IMMUTABLE PARALLEL SAFE SET search_path = pg_catalog, extensions, public AS $$ BEGIN - RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree', '='); + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_search', '='); END; $$ LANGUAGE plpgsql; CREATE OPERATOR = ( FUNCTION = {{prefix}}_internal.json_tree_blocked_compare_eq_doc_jsonb, - LEFTARG = public.{{prefix}}_json_tree, + LEFTARG = public.{{prefix}}_json_search, RIGHTARG = jsonb ); --! @brief Blocker: root document = (jsonb, document). --! @param a jsonb Native LHS operand. ---! @param b public.{{prefix}}_json_tree Right operand (encrypted payload). +--! @param b public.{{prefix}}_json_search Right operand (encrypted payload). --! @return boolean Never returns; always raises 'operator not supported'. -CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_compare_eq_jsonb_doc(a jsonb, b public.{{prefix}}_json_tree) +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_compare_eq_jsonb_doc(a jsonb, b public.{{prefix}}_json_search) RETURNS boolean IMMUTABLE PARALLEL SAFE SET search_path = pg_catalog, extensions, public AS $$ BEGIN - RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree', '='); + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_search', '='); END; $$ LANGUAGE plpgsql; CREATE OPERATOR = ( FUNCTION = {{prefix}}_internal.json_tree_blocked_compare_eq_jsonb_doc, LEFTARG = jsonb, - RIGHTARG = public.{{prefix}}_json_tree + RIGHTARG = public.{{prefix}}_json_search ); --! @brief Blocker: root document <> (document, document). ---! @param a public.{{prefix}}_json_tree Left operand (encrypted payload). ---! @param b public.{{prefix}}_json_tree Right operand (encrypted payload). +--! @param a public.{{prefix}}_json_search Left operand (encrypted payload). +--! @param b public.{{prefix}}_json_search Right operand (encrypted payload). --! @return boolean Never returns; always raises 'operator not supported'. -CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_compare_neq_doc_doc(a public.{{prefix}}_json_tree, b public.{{prefix}}_json_tree) +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_compare_neq_doc_doc(a public.{{prefix}}_json_search, b public.{{prefix}}_json_search) RETURNS boolean IMMUTABLE PARALLEL SAFE SET search_path = pg_catalog, extensions, public AS $$ BEGIN - RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree', '<>'); + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_search', '<>'); END; $$ LANGUAGE plpgsql; CREATE OPERATOR <> ( FUNCTION = {{prefix}}_internal.json_tree_blocked_compare_neq_doc_doc, - LEFTARG = public.{{prefix}}_json_tree, - RIGHTARG = public.{{prefix}}_json_tree + LEFTARG = public.{{prefix}}_json_search, + RIGHTARG = public.{{prefix}}_json_search ); --! @brief Blocker: root document <> (document, jsonb). ---! @param a public.{{prefix}}_json_tree Left operand (encrypted payload). +--! @param a public.{{prefix}}_json_search Left operand (encrypted payload). --! @param b jsonb Native RHS operand. --! @return boolean Never returns; always raises 'operator not supported'. -CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_compare_neq_doc_jsonb(a public.{{prefix}}_json_tree, b jsonb) +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_compare_neq_doc_jsonb(a public.{{prefix}}_json_search, b jsonb) RETURNS boolean IMMUTABLE PARALLEL SAFE SET search_path = pg_catalog, extensions, public AS $$ BEGIN - RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree', '<>'); + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_search', '<>'); END; $$ LANGUAGE plpgsql; CREATE OPERATOR <> ( FUNCTION = {{prefix}}_internal.json_tree_blocked_compare_neq_doc_jsonb, - LEFTARG = public.{{prefix}}_json_tree, + LEFTARG = public.{{prefix}}_json_search, RIGHTARG = jsonb ); --! @brief Blocker: root document <> (jsonb, document). --! @param a jsonb Native LHS operand. ---! @param b public.{{prefix}}_json_tree Right operand (encrypted payload). +--! @param b public.{{prefix}}_json_search Right operand (encrypted payload). --! @return boolean Never returns; always raises 'operator not supported'. -CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_compare_neq_jsonb_doc(a jsonb, b public.{{prefix}}_json_tree) +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_compare_neq_jsonb_doc(a jsonb, b public.{{prefix}}_json_search) RETURNS boolean IMMUTABLE PARALLEL SAFE SET search_path = pg_catalog, extensions, public AS $$ BEGIN - RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree', '<>'); + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_search', '<>'); END; $$ LANGUAGE plpgsql; CREATE OPERATOR <> ( FUNCTION = {{prefix}}_internal.json_tree_blocked_compare_neq_jsonb_doc, LEFTARG = jsonb, - RIGHTARG = public.{{prefix}}_json_tree + RIGHTARG = public.{{prefix}}_json_search ); --! @brief Blocker: root document < (document, document). ---! @param a public.{{prefix}}_json_tree Left operand (encrypted payload). ---! @param b public.{{prefix}}_json_tree Right operand (encrypted payload). +--! @param a public.{{prefix}}_json_search Left operand (encrypted payload). +--! @param b public.{{prefix}}_json_search Right operand (encrypted payload). --! @return boolean Never returns; always raises 'operator not supported'. -CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_compare_lt_doc_doc(a public.{{prefix}}_json_tree, b public.{{prefix}}_json_tree) +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_compare_lt_doc_doc(a public.{{prefix}}_json_search, b public.{{prefix}}_json_search) RETURNS boolean IMMUTABLE PARALLEL SAFE SET search_path = pg_catalog, extensions, public AS $$ BEGIN - RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree', '<'); + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_search', '<'); END; $$ LANGUAGE plpgsql; CREATE OPERATOR < ( FUNCTION = {{prefix}}_internal.json_tree_blocked_compare_lt_doc_doc, - LEFTARG = public.{{prefix}}_json_tree, - RIGHTARG = public.{{prefix}}_json_tree + LEFTARG = public.{{prefix}}_json_search, + RIGHTARG = public.{{prefix}}_json_search ); --! @brief Blocker: root document < (document, jsonb). ---! @param a public.{{prefix}}_json_tree Left operand (encrypted payload). +--! @param a public.{{prefix}}_json_search Left operand (encrypted payload). --! @param b jsonb Native RHS operand. --! @return boolean Never returns; always raises 'operator not supported'. -CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_compare_lt_doc_jsonb(a public.{{prefix}}_json_tree, b jsonb) +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_compare_lt_doc_jsonb(a public.{{prefix}}_json_search, b jsonb) RETURNS boolean IMMUTABLE PARALLEL SAFE SET search_path = pg_catalog, extensions, public AS $$ BEGIN - RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree', '<'); + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_search', '<'); END; $$ LANGUAGE plpgsql; CREATE OPERATOR < ( FUNCTION = {{prefix}}_internal.json_tree_blocked_compare_lt_doc_jsonb, - LEFTARG = public.{{prefix}}_json_tree, + LEFTARG = public.{{prefix}}_json_search, RIGHTARG = jsonb ); --! @brief Blocker: root document < (jsonb, document). --! @param a jsonb Native LHS operand. ---! @param b public.{{prefix}}_json_tree Right operand (encrypted payload). +--! @param b public.{{prefix}}_json_search Right operand (encrypted payload). --! @return boolean Never returns; always raises 'operator not supported'. -CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_compare_lt_jsonb_doc(a jsonb, b public.{{prefix}}_json_tree) +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_compare_lt_jsonb_doc(a jsonb, b public.{{prefix}}_json_search) RETURNS boolean IMMUTABLE PARALLEL SAFE SET search_path = pg_catalog, extensions, public AS $$ BEGIN - RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree', '<'); + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_search', '<'); END; $$ LANGUAGE plpgsql; CREATE OPERATOR < ( FUNCTION = {{prefix}}_internal.json_tree_blocked_compare_lt_jsonb_doc, LEFTARG = jsonb, - RIGHTARG = public.{{prefix}}_json_tree + RIGHTARG = public.{{prefix}}_json_search ); --! @brief Blocker: root document <= (document, document). ---! @param a public.{{prefix}}_json_tree Left operand (encrypted payload). ---! @param b public.{{prefix}}_json_tree Right operand (encrypted payload). +--! @param a public.{{prefix}}_json_search Left operand (encrypted payload). +--! @param b public.{{prefix}}_json_search Right operand (encrypted payload). --! @return boolean Never returns; always raises 'operator not supported'. -CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_compare_lte_doc_doc(a public.{{prefix}}_json_tree, b public.{{prefix}}_json_tree) +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_compare_lte_doc_doc(a public.{{prefix}}_json_search, b public.{{prefix}}_json_search) RETURNS boolean IMMUTABLE PARALLEL SAFE SET search_path = pg_catalog, extensions, public AS $$ BEGIN - RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree', '<='); + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_search', '<='); END; $$ LANGUAGE plpgsql; CREATE OPERATOR <= ( FUNCTION = {{prefix}}_internal.json_tree_blocked_compare_lte_doc_doc, - LEFTARG = public.{{prefix}}_json_tree, - RIGHTARG = public.{{prefix}}_json_tree + LEFTARG = public.{{prefix}}_json_search, + RIGHTARG = public.{{prefix}}_json_search ); --! @brief Blocker: root document <= (document, jsonb). ---! @param a public.{{prefix}}_json_tree Left operand (encrypted payload). +--! @param a public.{{prefix}}_json_search Left operand (encrypted payload). --! @param b jsonb Native RHS operand. --! @return boolean Never returns; always raises 'operator not supported'. -CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_compare_lte_doc_jsonb(a public.{{prefix}}_json_tree, b jsonb) +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_compare_lte_doc_jsonb(a public.{{prefix}}_json_search, b jsonb) RETURNS boolean IMMUTABLE PARALLEL SAFE SET search_path = pg_catalog, extensions, public AS $$ BEGIN - RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree', '<='); + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_search', '<='); END; $$ LANGUAGE plpgsql; CREATE OPERATOR <= ( FUNCTION = {{prefix}}_internal.json_tree_blocked_compare_lte_doc_jsonb, - LEFTARG = public.{{prefix}}_json_tree, + LEFTARG = public.{{prefix}}_json_search, RIGHTARG = jsonb ); --! @brief Blocker: root document <= (jsonb, document). --! @param a jsonb Native LHS operand. ---! @param b public.{{prefix}}_json_tree Right operand (encrypted payload). +--! @param b public.{{prefix}}_json_search Right operand (encrypted payload). --! @return boolean Never returns; always raises 'operator not supported'. -CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_compare_lte_jsonb_doc(a jsonb, b public.{{prefix}}_json_tree) +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_compare_lte_jsonb_doc(a jsonb, b public.{{prefix}}_json_search) RETURNS boolean IMMUTABLE PARALLEL SAFE SET search_path = pg_catalog, extensions, public AS $$ BEGIN - RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree', '<='); + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_search', '<='); END; $$ LANGUAGE plpgsql; CREATE OPERATOR <= ( FUNCTION = {{prefix}}_internal.json_tree_blocked_compare_lte_jsonb_doc, LEFTARG = jsonb, - RIGHTARG = public.{{prefix}}_json_tree + RIGHTARG = public.{{prefix}}_json_search ); --! @brief Blocker: root document > (document, document). ---! @param a public.{{prefix}}_json_tree Left operand (encrypted payload). ---! @param b public.{{prefix}}_json_tree Right operand (encrypted payload). +--! @param a public.{{prefix}}_json_search Left operand (encrypted payload). +--! @param b public.{{prefix}}_json_search Right operand (encrypted payload). --! @return boolean Never returns; always raises 'operator not supported'. -CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_compare_gt_doc_doc(a public.{{prefix}}_json_tree, b public.{{prefix}}_json_tree) +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_compare_gt_doc_doc(a public.{{prefix}}_json_search, b public.{{prefix}}_json_search) RETURNS boolean IMMUTABLE PARALLEL SAFE SET search_path = pg_catalog, extensions, public AS $$ BEGIN - RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree', '>'); + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_search', '>'); END; $$ LANGUAGE plpgsql; CREATE OPERATOR > ( FUNCTION = {{prefix}}_internal.json_tree_blocked_compare_gt_doc_doc, - LEFTARG = public.{{prefix}}_json_tree, - RIGHTARG = public.{{prefix}}_json_tree + LEFTARG = public.{{prefix}}_json_search, + RIGHTARG = public.{{prefix}}_json_search ); --! @brief Blocker: root document > (document, jsonb). ---! @param a public.{{prefix}}_json_tree Left operand (encrypted payload). +--! @param a public.{{prefix}}_json_search Left operand (encrypted payload). --! @param b jsonb Native RHS operand. --! @return boolean Never returns; always raises 'operator not supported'. -CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_compare_gt_doc_jsonb(a public.{{prefix}}_json_tree, b jsonb) +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_compare_gt_doc_jsonb(a public.{{prefix}}_json_search, b jsonb) RETURNS boolean IMMUTABLE PARALLEL SAFE SET search_path = pg_catalog, extensions, public AS $$ BEGIN - RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree', '>'); + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_search', '>'); END; $$ LANGUAGE plpgsql; CREATE OPERATOR > ( FUNCTION = {{prefix}}_internal.json_tree_blocked_compare_gt_doc_jsonb, - LEFTARG = public.{{prefix}}_json_tree, + LEFTARG = public.{{prefix}}_json_search, RIGHTARG = jsonb ); --! @brief Blocker: root document > (jsonb, document). --! @param a jsonb Native LHS operand. ---! @param b public.{{prefix}}_json_tree Right operand (encrypted payload). +--! @param b public.{{prefix}}_json_search Right operand (encrypted payload). --! @return boolean Never returns; always raises 'operator not supported'. -CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_compare_gt_jsonb_doc(a jsonb, b public.{{prefix}}_json_tree) +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_compare_gt_jsonb_doc(a jsonb, b public.{{prefix}}_json_search) RETURNS boolean IMMUTABLE PARALLEL SAFE SET search_path = pg_catalog, extensions, public AS $$ BEGIN - RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree', '>'); + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_search', '>'); END; $$ LANGUAGE plpgsql; CREATE OPERATOR > ( FUNCTION = {{prefix}}_internal.json_tree_blocked_compare_gt_jsonb_doc, LEFTARG = jsonb, - RIGHTARG = public.{{prefix}}_json_tree + RIGHTARG = public.{{prefix}}_json_search ); --! @brief Blocker: root document >= (document, document). ---! @param a public.{{prefix}}_json_tree Left operand (encrypted payload). ---! @param b public.{{prefix}}_json_tree Right operand (encrypted payload). +--! @param a public.{{prefix}}_json_search Left operand (encrypted payload). +--! @param b public.{{prefix}}_json_search Right operand (encrypted payload). --! @return boolean Never returns; always raises 'operator not supported'. -CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_compare_gte_doc_doc(a public.{{prefix}}_json_tree, b public.{{prefix}}_json_tree) +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_compare_gte_doc_doc(a public.{{prefix}}_json_search, b public.{{prefix}}_json_search) RETURNS boolean IMMUTABLE PARALLEL SAFE SET search_path = pg_catalog, extensions, public AS $$ BEGIN - RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree', '>='); + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_search', '>='); END; $$ LANGUAGE plpgsql; CREATE OPERATOR >= ( FUNCTION = {{prefix}}_internal.json_tree_blocked_compare_gte_doc_doc, - LEFTARG = public.{{prefix}}_json_tree, - RIGHTARG = public.{{prefix}}_json_tree + LEFTARG = public.{{prefix}}_json_search, + RIGHTARG = public.{{prefix}}_json_search ); --! @brief Blocker: root document >= (document, jsonb). ---! @param a public.{{prefix}}_json_tree Left operand (encrypted payload). +--! @param a public.{{prefix}}_json_search Left operand (encrypted payload). --! @param b jsonb Native RHS operand. --! @return boolean Never returns; always raises 'operator not supported'. -CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_compare_gte_doc_jsonb(a public.{{prefix}}_json_tree, b jsonb) +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_compare_gte_doc_jsonb(a public.{{prefix}}_json_search, b jsonb) RETURNS boolean IMMUTABLE PARALLEL SAFE SET search_path = pg_catalog, extensions, public AS $$ BEGIN - RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree', '>='); + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_search', '>='); END; $$ LANGUAGE plpgsql; CREATE OPERATOR >= ( FUNCTION = {{prefix}}_internal.json_tree_blocked_compare_gte_doc_jsonb, - LEFTARG = public.{{prefix}}_json_tree, + LEFTARG = public.{{prefix}}_json_search, RIGHTARG = jsonb ); --! @brief Blocker: root document >= (jsonb, document). --! @param a jsonb Native LHS operand. ---! @param b public.{{prefix}}_json_tree Right operand (encrypted payload). +--! @param b public.{{prefix}}_json_search Right operand (encrypted payload). --! @return boolean Never returns; always raises 'operator not supported'. -CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_compare_gte_jsonb_doc(a jsonb, b public.{{prefix}}_json_tree) +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_compare_gte_jsonb_doc(a jsonb, b public.{{prefix}}_json_search) RETURNS boolean IMMUTABLE PARALLEL SAFE SET search_path = pg_catalog, extensions, public AS $$ BEGIN - RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree', '>='); + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_search', '>='); END; $$ LANGUAGE plpgsql; CREATE OPERATOR >= ( FUNCTION = {{prefix}}_internal.json_tree_blocked_compare_gte_jsonb_doc, LEFTARG = jsonb, - RIGHTARG = public.{{prefix}}_json_tree + RIGHTARG = public.{{prefix}}_json_search ); --! @brief Blocker: = between two extracted entries (no exact equality term). ---! @param a public.{{prefix}}_json_tree_entry Left operand (entry). ---! @param b public.{{prefix}}_json_tree_entry Right operand (entry). +--! @param a public.{{prefix}}_json_entry Left operand (entry). +--! @param b public.{{prefix}}_json_entry Right operand (entry). --! @return boolean Never returns; always raises 'operator not supported'. -CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_entry_eq(a public.{{prefix}}_json_tree_entry, b public.{{prefix}}_json_tree_entry) +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_entry_eq(a public.{{prefix}}_json_entry, b public.{{prefix}}_json_entry) RETURNS boolean IMMUTABLE PARALLEL SAFE SET search_path = pg_catalog, extensions, public AS $$ BEGIN - RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree_entry', '='); + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_entry', '='); END; $$ LANGUAGE plpgsql; CREATE OPERATOR = ( FUNCTION = {{prefix}}_internal.json_tree_blocked_entry_eq, - LEFTARG = public.{{prefix}}_json_tree_entry, - RIGHTARG = public.{{prefix}}_json_tree_entry + LEFTARG = public.{{prefix}}_json_entry, + RIGHTARG = public.{{prefix}}_json_entry ); --! @brief Blocker: <> between two extracted entries. ---! @param a public.{{prefix}}_json_tree_entry Left operand (entry). ---! @param b public.{{prefix}}_json_tree_entry Right operand (entry). +--! @param a public.{{prefix}}_json_entry Left operand (entry). +--! @param b public.{{prefix}}_json_entry Right operand (entry). --! @return boolean Never returns; always raises 'operator not supported'. -CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_entry_neq(a public.{{prefix}}_json_tree_entry, b public.{{prefix}}_json_tree_entry) +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_entry_neq(a public.{{prefix}}_json_entry, b public.{{prefix}}_json_entry) RETURNS boolean IMMUTABLE PARALLEL SAFE SET search_path = pg_catalog, extensions, public AS $$ BEGIN - RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree_entry', '<>'); + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_entry', '<>'); END; $$ LANGUAGE plpgsql; CREATE OPERATOR <> ( FUNCTION = {{prefix}}_internal.json_tree_blocked_entry_neq, - LEFTARG = public.{{prefix}}_json_tree_entry, - RIGHTARG = public.{{prefix}}_json_tree_entry + LEFTARG = public.{{prefix}}_json_entry, + RIGHTARG = public.{{prefix}}_json_entry ); --! @brief Blocker: < (entry, jsonb): an ordering operand is typed. --! ---! An entry compares with `{{prefix}}.query_json_tree_leaf` or another +--! An entry compares with `{{prefix}}.query_json_leaf` or another --! entry. Against plain jsonb, the native operator would compare the entry's --! ciphertext object, which is what a query that forgets the operand's cast --! would silently get. ---! @param a public.{{prefix}}_json_tree_entry Left operand (entry). +--! @param a public.{{prefix}}_json_entry Left operand (entry). --! @param b jsonb Native RHS operand. --! @return boolean Never returns; always raises 'operator not supported'. -CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_entry_lt_jsonb(a public.{{prefix}}_json_tree_entry, b jsonb) +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_entry_lt_jsonb(a public.{{prefix}}_json_entry, b jsonb) RETURNS boolean IMMUTABLE PARALLEL SAFE SET search_path = pg_catalog, extensions, public AS $$ BEGIN - RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree_entry', '<'); + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_entry', '<'); END; $$ LANGUAGE plpgsql; CREATE OPERATOR < ( FUNCTION = {{prefix}}_internal.json_tree_blocked_entry_lt_jsonb, - LEFTARG = public.{{prefix}}_json_tree_entry, + LEFTARG = public.{{prefix}}_json_entry, RIGHTARG = jsonb ); --! @brief Blocker: < (jsonb, entry). --! @param a jsonb Native LHS operand. ---! @param b public.{{prefix}}_json_tree_entry Right operand (entry). +--! @param b public.{{prefix}}_json_entry Right operand (entry). --! @return boolean Never returns; always raises 'operator not supported'. -CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_jsonb_lt_entry(a jsonb, b public.{{prefix}}_json_tree_entry) +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_jsonb_lt_entry(a jsonb, b public.{{prefix}}_json_entry) RETURNS boolean IMMUTABLE PARALLEL SAFE SET search_path = pg_catalog, extensions, public AS $$ BEGIN - RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree_entry', '<'); + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_entry', '<'); END; $$ LANGUAGE plpgsql; CREATE OPERATOR < ( FUNCTION = {{prefix}}_internal.json_tree_blocked_jsonb_lt_entry, LEFTARG = jsonb, - RIGHTARG = public.{{prefix}}_json_tree_entry + RIGHTARG = public.{{prefix}}_json_entry ); --! @brief Blocker: <= (entry, jsonb): an ordering operand is typed. --! ---! An entry compares with `{{prefix}}.query_json_tree_leaf` or another +--! An entry compares with `{{prefix}}.query_json_leaf` or another --! entry. Against plain jsonb, the native operator would compare the entry's --! ciphertext object, which is what a query that forgets the operand's cast --! would silently get. ---! @param a public.{{prefix}}_json_tree_entry Left operand (entry). +--! @param a public.{{prefix}}_json_entry Left operand (entry). --! @param b jsonb Native RHS operand. --! @return boolean Never returns; always raises 'operator not supported'. -CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_entry_lte_jsonb(a public.{{prefix}}_json_tree_entry, b jsonb) +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_entry_lte_jsonb(a public.{{prefix}}_json_entry, b jsonb) RETURNS boolean IMMUTABLE PARALLEL SAFE SET search_path = pg_catalog, extensions, public AS $$ BEGIN - RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree_entry', '<='); + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_entry', '<='); END; $$ LANGUAGE plpgsql; CREATE OPERATOR <= ( FUNCTION = {{prefix}}_internal.json_tree_blocked_entry_lte_jsonb, - LEFTARG = public.{{prefix}}_json_tree_entry, + LEFTARG = public.{{prefix}}_json_entry, RIGHTARG = jsonb ); --! @brief Blocker: <= (jsonb, entry). --! @param a jsonb Native LHS operand. ---! @param b public.{{prefix}}_json_tree_entry Right operand (entry). +--! @param b public.{{prefix}}_json_entry Right operand (entry). --! @return boolean Never returns; always raises 'operator not supported'. -CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_jsonb_lte_entry(a jsonb, b public.{{prefix}}_json_tree_entry) +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_jsonb_lte_entry(a jsonb, b public.{{prefix}}_json_entry) RETURNS boolean IMMUTABLE PARALLEL SAFE SET search_path = pg_catalog, extensions, public AS $$ BEGIN - RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree_entry', '<='); + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_entry', '<='); END; $$ LANGUAGE plpgsql; CREATE OPERATOR <= ( FUNCTION = {{prefix}}_internal.json_tree_blocked_jsonb_lte_entry, LEFTARG = jsonb, - RIGHTARG = public.{{prefix}}_json_tree_entry + RIGHTARG = public.{{prefix}}_json_entry ); --! @brief Blocker: > (entry, jsonb): an ordering operand is typed. --! ---! An entry compares with `{{prefix}}.query_json_tree_leaf` or another +--! An entry compares with `{{prefix}}.query_json_leaf` or another --! entry. Against plain jsonb, the native operator would compare the entry's --! ciphertext object, which is what a query that forgets the operand's cast --! would silently get. ---! @param a public.{{prefix}}_json_tree_entry Left operand (entry). +--! @param a public.{{prefix}}_json_entry Left operand (entry). --! @param b jsonb Native RHS operand. --! @return boolean Never returns; always raises 'operator not supported'. -CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_entry_gt_jsonb(a public.{{prefix}}_json_tree_entry, b jsonb) +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_entry_gt_jsonb(a public.{{prefix}}_json_entry, b jsonb) RETURNS boolean IMMUTABLE PARALLEL SAFE SET search_path = pg_catalog, extensions, public AS $$ BEGIN - RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree_entry', '>'); + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_entry', '>'); END; $$ LANGUAGE plpgsql; CREATE OPERATOR > ( FUNCTION = {{prefix}}_internal.json_tree_blocked_entry_gt_jsonb, - LEFTARG = public.{{prefix}}_json_tree_entry, + LEFTARG = public.{{prefix}}_json_entry, RIGHTARG = jsonb ); --! @brief Blocker: > (jsonb, entry). --! @param a jsonb Native LHS operand. ---! @param b public.{{prefix}}_json_tree_entry Right operand (entry). +--! @param b public.{{prefix}}_json_entry Right operand (entry). --! @return boolean Never returns; always raises 'operator not supported'. -CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_jsonb_gt_entry(a jsonb, b public.{{prefix}}_json_tree_entry) +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_jsonb_gt_entry(a jsonb, b public.{{prefix}}_json_entry) RETURNS boolean IMMUTABLE PARALLEL SAFE SET search_path = pg_catalog, extensions, public AS $$ BEGIN - RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree_entry', '>'); + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_entry', '>'); END; $$ LANGUAGE plpgsql; CREATE OPERATOR > ( FUNCTION = {{prefix}}_internal.json_tree_blocked_jsonb_gt_entry, LEFTARG = jsonb, - RIGHTARG = public.{{prefix}}_json_tree_entry + RIGHTARG = public.{{prefix}}_json_entry ); --! @brief Blocker: >= (entry, jsonb): an ordering operand is typed. --! ---! An entry compares with `{{prefix}}.query_json_tree_leaf` or another +--! An entry compares with `{{prefix}}.query_json_leaf` or another --! entry. Against plain jsonb, the native operator would compare the entry's --! ciphertext object, which is what a query that forgets the operand's cast --! would silently get. ---! @param a public.{{prefix}}_json_tree_entry Left operand (entry). +--! @param a public.{{prefix}}_json_entry Left operand (entry). --! @param b jsonb Native RHS operand. --! @return boolean Never returns; always raises 'operator not supported'. -CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_entry_gte_jsonb(a public.{{prefix}}_json_tree_entry, b jsonb) +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_entry_gte_jsonb(a public.{{prefix}}_json_entry, b jsonb) RETURNS boolean IMMUTABLE PARALLEL SAFE SET search_path = pg_catalog, extensions, public AS $$ BEGIN - RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree_entry', '>='); + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_entry', '>='); END; $$ LANGUAGE plpgsql; CREATE OPERATOR >= ( FUNCTION = {{prefix}}_internal.json_tree_blocked_entry_gte_jsonb, - LEFTARG = public.{{prefix}}_json_tree_entry, + LEFTARG = public.{{prefix}}_json_entry, RIGHTARG = jsonb ); --! @brief Blocker: >= (jsonb, entry). --! @param a jsonb Native LHS operand. ---! @param b public.{{prefix}}_json_tree_entry Right operand (entry). +--! @param b public.{{prefix}}_json_entry Right operand (entry). --! @return boolean Never returns; always raises 'operator not supported'. -CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_jsonb_gte_entry(a jsonb, b public.{{prefix}}_json_tree_entry) +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_jsonb_gte_entry(a jsonb, b public.{{prefix}}_json_entry) RETURNS boolean IMMUTABLE PARALLEL SAFE SET search_path = pg_catalog, extensions, public AS $$ BEGIN - RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree_entry', '>='); + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_entry', '>='); END; $$ LANGUAGE plpgsql; CREATE OPERATOR >= ( FUNCTION = {{prefix}}_internal.json_tree_blocked_jsonb_gte_entry, LEFTARG = jsonb, - RIGHTARG = public.{{prefix}}_json_tree_entry + RIGHTARG = public.{{prefix}}_json_entry ); --! @brief Blocker: = (entry, jsonb): an ordering operand is typed. --! ---! An entry compares with `{{prefix}}.query_json_tree_leaf` or another +--! An entry compares with `{{prefix}}.query_json_leaf` or another --! entry. Against plain jsonb, the native operator would compare the entry's --! ciphertext object, which is what a query that forgets the operand's cast --! would silently get. ---! @param a public.{{prefix}}_json_tree_entry Left operand (entry). +--! @param a public.{{prefix}}_json_entry Left operand (entry). --! @param b jsonb Native RHS operand. --! @return boolean Never returns; always raises 'operator not supported'. -CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_entry_eq_jsonb(a public.{{prefix}}_json_tree_entry, b jsonb) +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_entry_eq_jsonb(a public.{{prefix}}_json_entry, b jsonb) RETURNS boolean IMMUTABLE PARALLEL SAFE SET search_path = pg_catalog, extensions, public AS $$ BEGIN - RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree_entry', '='); + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_entry', '='); END; $$ LANGUAGE plpgsql; CREATE OPERATOR = ( FUNCTION = {{prefix}}_internal.json_tree_blocked_entry_eq_jsonb, - LEFTARG = public.{{prefix}}_json_tree_entry, + LEFTARG = public.{{prefix}}_json_entry, RIGHTARG = jsonb ); --! @brief Blocker: = (jsonb, entry). --! @param a jsonb Native LHS operand. ---! @param b public.{{prefix}}_json_tree_entry Right operand (entry). +--! @param b public.{{prefix}}_json_entry Right operand (entry). --! @return boolean Never returns; always raises 'operator not supported'. -CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_jsonb_eq_entry(a jsonb, b public.{{prefix}}_json_tree_entry) +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_jsonb_eq_entry(a jsonb, b public.{{prefix}}_json_entry) RETURNS boolean IMMUTABLE PARALLEL SAFE SET search_path = pg_catalog, extensions, public AS $$ BEGIN - RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree_entry', '='); + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_entry', '='); END; $$ LANGUAGE plpgsql; CREATE OPERATOR = ( FUNCTION = {{prefix}}_internal.json_tree_blocked_jsonb_eq_entry, LEFTARG = jsonb, - RIGHTARG = public.{{prefix}}_json_tree_entry + RIGHTARG = public.{{prefix}}_json_entry ); --! @brief Blocker: <> (entry, jsonb): an ordering operand is typed. --! ---! An entry compares with `{{prefix}}.query_json_tree_leaf` or another +--! An entry compares with `{{prefix}}.query_json_leaf` or another --! entry. Against plain jsonb, the native operator would compare the entry's --! ciphertext object, which is what a query that forgets the operand's cast --! would silently get. ---! @param a public.{{prefix}}_json_tree_entry Left operand (entry). +--! @param a public.{{prefix}}_json_entry Left operand (entry). --! @param b jsonb Native RHS operand. --! @return boolean Never returns; always raises 'operator not supported'. -CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_entry_neq_jsonb(a public.{{prefix}}_json_tree_entry, b jsonb) +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_entry_neq_jsonb(a public.{{prefix}}_json_entry, b jsonb) RETURNS boolean IMMUTABLE PARALLEL SAFE SET search_path = pg_catalog, extensions, public AS $$ BEGIN - RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree_entry', '<>'); + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_entry', '<>'); END; $$ LANGUAGE plpgsql; CREATE OPERATOR <> ( FUNCTION = {{prefix}}_internal.json_tree_blocked_entry_neq_jsonb, - LEFTARG = public.{{prefix}}_json_tree_entry, + LEFTARG = public.{{prefix}}_json_entry, RIGHTARG = jsonb ); --! @brief Blocker: <> (jsonb, entry). --! @param a jsonb Native LHS operand. ---! @param b public.{{prefix}}_json_tree_entry Right operand (entry). +--! @param b public.{{prefix}}_json_entry Right operand (entry). --! @return boolean Never returns; always raises 'operator not supported'. -CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_jsonb_neq_entry(a jsonb, b public.{{prefix}}_json_tree_entry) +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_jsonb_neq_entry(a jsonb, b public.{{prefix}}_json_entry) RETURNS boolean IMMUTABLE PARALLEL SAFE SET search_path = pg_catalog, extensions, public AS $$ BEGIN - RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree_entry', '<>'); + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_entry', '<>'); END; $$ LANGUAGE plpgsql; CREATE OPERATOR <> ( FUNCTION = {{prefix}}_internal.json_tree_blocked_jsonb_neq_entry, LEFTARG = jsonb, - RIGHTARG = public.{{prefix}}_json_tree_entry + RIGHTARG = public.{{prefix}}_json_entry ); --! @brief Blocker: @> (document, entry): an entry is not a needle. ---! @param a public.{{prefix}}_json_tree Left operand (encrypted payload). ---! @param b public.{{prefix}}_json_tree_entry Right operand (entry). +--! @param a public.{{prefix}}_json_search Left operand (encrypted payload). +--! @param b public.{{prefix}}_json_entry Right operand (entry). --! @return boolean Never returns; always raises 'operator not supported'. -CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_contains_entry(a public.{{prefix}}_json_tree, b public.{{prefix}}_json_tree_entry) +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_contains_entry(a public.{{prefix}}_json_search, b public.{{prefix}}_json_entry) RETURNS boolean IMMUTABLE PARALLEL SAFE SET search_path = pg_catalog, extensions, public AS $$ BEGIN - RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree', '@>'); + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_search', '@>'); END; $$ LANGUAGE plpgsql; CREATE OPERATOR @> ( FUNCTION = {{prefix}}_internal.json_tree_blocked_contains_entry, - LEFTARG = public.{{prefix}}_json_tree, - RIGHTARG = public.{{prefix}}_json_tree_entry + LEFTARG = public.{{prefix}}_json_search, + RIGHTARG = public.{{prefix}}_json_entry ); --! @brief Blocker: <@ (entry, document). ---! @param a public.{{prefix}}_json_tree_entry Left operand (entry). ---! @param b public.{{prefix}}_json_tree Right operand (encrypted payload). +--! @param a public.{{prefix}}_json_entry Left operand (entry). +--! @param b public.{{prefix}}_json_search Right operand (encrypted payload). --! @return boolean Never returns; always raises 'operator not supported'. -CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_contained_by_entry(a public.{{prefix}}_json_tree_entry, b public.{{prefix}}_json_tree) +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_contained_by_entry(a public.{{prefix}}_json_entry, b public.{{prefix}}_json_search) RETURNS boolean IMMUTABLE PARALLEL SAFE SET search_path = pg_catalog, extensions, public AS $$ BEGIN - RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree', '<@'); + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_search', '<@'); END; $$ LANGUAGE plpgsql; CREATE OPERATOR <@ ( FUNCTION = {{prefix}}_internal.json_tree_blocked_contained_by_entry, - LEFTARG = public.{{prefix}}_json_tree_entry, - RIGHTARG = public.{{prefix}}_json_tree + LEFTARG = public.{{prefix}}_json_entry, + RIGHTARG = public.{{prefix}}_json_search ); --! @brief Blocker: @> (document, jsonb): a needle must be typed. ---! @param a public.{{prefix}}_json_tree Left operand (encrypted payload). +--! @param a public.{{prefix}}_json_search Left operand (encrypted payload). --! @param b jsonb Native RHS operand. --! @return boolean Never returns; always raises 'operator not supported'. -CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_contains_jsonb(a public.{{prefix}}_json_tree, b jsonb) +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_contains_jsonb(a public.{{prefix}}_json_search, b jsonb) RETURNS boolean IMMUTABLE PARALLEL SAFE SET search_path = pg_catalog, extensions, public AS $$ BEGIN - RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree', '@>'); + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_search', '@>'); END; $$ LANGUAGE plpgsql; CREATE OPERATOR @> ( FUNCTION = {{prefix}}_internal.json_tree_blocked_contains_jsonb, - LEFTARG = public.{{prefix}}_json_tree, + LEFTARG = public.{{prefix}}_json_search, RIGHTARG = jsonb ); --! @brief Blocker: @> (jsonb, document). --! @param a jsonb Native LHS operand. ---! @param b public.{{prefix}}_json_tree Right operand (encrypted payload). +--! @param b public.{{prefix}}_json_search Right operand (encrypted payload). --! @return boolean Never returns; always raises 'operator not supported'. -CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_contains_jsonb_lhs(a jsonb, b public.{{prefix}}_json_tree) +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_contains_jsonb_lhs(a jsonb, b public.{{prefix}}_json_search) RETURNS boolean IMMUTABLE PARALLEL SAFE SET search_path = pg_catalog, extensions, public AS $$ BEGIN - RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree', '@>'); + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_search', '@>'); END; $$ LANGUAGE plpgsql; CREATE OPERATOR @> ( FUNCTION = {{prefix}}_internal.json_tree_blocked_contains_jsonb_lhs, LEFTARG = jsonb, - RIGHTARG = public.{{prefix}}_json_tree + RIGHTARG = public.{{prefix}}_json_search ); --! @brief Blocker: <@ (document, jsonb). ---! @param a public.{{prefix}}_json_tree Left operand (encrypted payload). +--! @param a public.{{prefix}}_json_search Left operand (encrypted payload). --! @param b jsonb Native RHS operand. --! @return boolean Never returns; always raises 'operator not supported'. -CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_contained_by_jsonb(a public.{{prefix}}_json_tree, b jsonb) +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_contained_by_jsonb(a public.{{prefix}}_json_search, b jsonb) RETURNS boolean IMMUTABLE PARALLEL SAFE SET search_path = pg_catalog, extensions, public AS $$ BEGIN - RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree', '<@'); + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_search', '<@'); END; $$ LANGUAGE plpgsql; CREATE OPERATOR <@ ( FUNCTION = {{prefix}}_internal.json_tree_blocked_contained_by_jsonb, - LEFTARG = public.{{prefix}}_json_tree, + LEFTARG = public.{{prefix}}_json_search, RIGHTARG = jsonb ); --! @brief Blocker: <@ (jsonb, document). --! @param a jsonb Native LHS operand. ---! @param b public.{{prefix}}_json_tree Right operand (encrypted payload). +--! @param b public.{{prefix}}_json_search Right operand (encrypted payload). --! @return boolean Never returns; always raises 'operator not supported'. -CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_contained_by_jsonb_lhs(a jsonb, b public.{{prefix}}_json_tree) +CREATE FUNCTION {{prefix}}_internal.json_tree_blocked_contained_by_jsonb_lhs(a jsonb, b public.{{prefix}}_json_search) RETURNS boolean IMMUTABLE PARALLEL SAFE SET search_path = pg_catalog, extensions, public AS $$ BEGIN - RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_tree', '<@'); + RETURN {{prefix}}_internal.encrypted_domain_unsupported_bool('public.{{prefix}}_json_search', '<@'); END; $$ LANGUAGE plpgsql; CREATE OPERATOR <@ ( FUNCTION = {{prefix}}_internal.json_tree_blocked_contained_by_jsonb_lhs, LEFTARG = jsonb, - RIGHTARG = public.{{prefix}}_json_tree + RIGHTARG = public.{{prefix}}_json_search ); diff --git a/packages/eql/src/v3/json_tree/functions.sql b/packages/eql/src/v3/json_tree/functions.sql index 5951aa0e7..c2e6b1413 100644 --- a/packages/eql/src/v3/json_tree/functions.sql +++ b/packages/eql/src/v3/json_tree/functions.sql @@ -199,7 +199,7 @@ $$; --! `{{prefix}}_internal.ope_cllw(jsonb)` reads hex, so the family --! has its own extractor. --! @internal ---! @param val jsonb An entry or a `query_json_tree_leaf`, as jsonb. +--! @param val jsonb An entry or a `query_json_leaf`, as jsonb. --! @return {{prefix}}_internal.ope_cllw The term; NULL when there is none. CREATE FUNCTION {{prefix}}_internal.json_tree_ope_cllw(val jsonb) RETURNS {{prefix}}_internal.ope_cllw @@ -272,12 +272,12 @@ COMMENT ON FUNCTION {{prefix}}.json_tree_meta(jsonb) IS --! @param val jsonb The document. --! @param elem jsonb The entry, as it sits in `sv`. --! @param j integer The entry's ordinal in `sv`, from zero. ---! @return public.{{prefix}}_json_tree_entry The entry with `i`, `v`, `h` and `j`. +--! @return public.{{prefix}}_json_entry The entry with `i`, `v`, `h` and `j`. CREATE FUNCTION {{prefix}}_internal.json_tree_graft(val jsonb, elem jsonb, j integer) - RETURNS public.{{prefix}}_json_tree_entry + RETURNS public.{{prefix}}_json_entry LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE AS $$ - SELECT ({{prefix}}.json_tree_meta(val) || elem || jsonb_build_object('j', j))::public.{{prefix}}_json_tree_entry + SELECT ({{prefix}}.json_tree_meta(val) || elem || jsonb_build_object('j', j))::public.{{prefix}}_json_entry $$; ------------------------------------------------------------------------------ @@ -292,10 +292,10 @@ $$; --! --! @param val jsonb The document. --! @param selector text The path selector, base64. ---! @return SETOF public.{{prefix}}_json_tree_entry The matching entries, in +--! @return SETOF public.{{prefix}}_json_entry The matching entries, in --! document order. CREATE FUNCTION {{prefix}}.json_tree_path_query(val jsonb, selector text) - RETURNS SETOF public.{{prefix}}_json_tree_entry + RETURNS SETOF public.{{prefix}}_json_entry LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE AS $$ SELECT {{prefix}}_internal.json_tree_graft(val, elem, (ord - 1)::integer) @@ -318,10 +318,10 @@ COMMENT ON FUNCTION {{prefix}}.json_tree_path_query(jsonb, text) IS --! --! @param val jsonb The document. --! @param selector text The path selector, base64. ---! @return public.{{prefix}}_json_tree_entry The entry, or NULL when the document +--! @return public.{{prefix}}_json_entry The entry, or NULL when the document --! has no node at that path. CREATE FUNCTION {{prefix}}.json_tree_path_query_first(val jsonb, selector text) - RETURNS public.{{prefix}}_json_tree_entry + RETURNS public.{{prefix}}_json_entry LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE COST 1000 AS $$ @@ -357,7 +357,7 @@ COMMENT ON FUNCTION {{prefix}}.json_tree_path_exists(jsonb, text) IS ------------------------------------------------------------------------------ --! @brief The stored document's containment projection: the needle shape ---! `{{prefix}}.query_json_tree` holds, as the client wrote it in `q`. +--! `{{prefix}}.query_json` holds, as the client wrote it in `q`. --! --! The projection is the document's `{s, vs}` pairs nested by array --! element, `{"sv": [{"s", "vs"}, …], "el": [group, …]}`: the pairs outside @@ -374,8 +374,8 @@ COMMENT ON FUNCTION {{prefix}}.json_tree_path_exists(jsonb, text) IS --! the index engages; GIN is lossy and the executor rechecks the real --! operator on the heap tuple, which is the exact containment. --! ---! @param e public.{{prefix}}_json_tree The stored document. ---! @return jsonb Its projection, the shape of `{{prefix}}.query_json_tree`. +--! @param e public.{{prefix}}_json_search The stored document. +--! @return jsonb Its projection, the shape of `{{prefix}}.query_json`. --! @note Returns raw jsonb rather than the needle domain: a domain cast --! runs the needle's recursive CHECK, and the recheck would pay it --! once per candidate row. The document's own CHECK validated `q` @@ -388,7 +388,7 @@ COMMENT ON FUNCTION {{prefix}}.json_tree_path_exists(jsonb, text) IS --! decompressed to read `q`. Un-inlined, the call keeps its cost, the --! index expression still matches it, and the planner takes the --! index. The call itself is a few microseconds per recheck row. -CREATE FUNCTION {{prefix}}.json_tree_query(e public.{{prefix}}_json_tree) +CREATE FUNCTION {{prefix}}.json_tree_query(e public.{{prefix}}_json_search) RETURNS jsonb LANGUAGE plpgsql IMMUTABLE STRICT PARALLEL SAFE COST 1000 @@ -405,8 +405,8 @@ $$; --! probes are over `{{prefix}}.json_tree_query(a)`, so a GIN index on that --! expression serves them as one bitmap OR. --! ---! @param a public.{{prefix}}_json_tree The stored document. ---! @param b {{prefix}}.query_json_tree The needle. +--! @param a public.{{prefix}}_json_search The stored document. +--! @param b {{prefix}}.query_json The needle. --! @return boolean True when `a` contains `b` as `jsonb @>` would contain the --! plaintexts; NULL when either is NULL. --! @note Deliberately not STRICT. The planner inlines a STRICT SQL function @@ -415,7 +415,7 @@ $$; --! filter and the GIN index never engages. The body returns NULL for --! a NULL argument on its own: a NULL side of `@>` is NULL, and NULL --! OR NULL is NULL. -CREATE FUNCTION {{prefix}}.json_tree_contains(a public.{{prefix}}_json_tree, b {{prefix}}.query_json_tree) +CREATE FUNCTION {{prefix}}.json_tree_contains(a public.{{prefix}}_json_search, b {{prefix}}.query_json) RETURNS boolean LANGUAGE sql IMMUTABLE PARALLEL SAFE AS $$ @@ -429,10 +429,10 @@ $$; --! so a stored root scalar inside a needle root array is the one case this --! answers false where `jsonb <@` answers true. --! ---! @param a public.{{prefix}}_json_tree The stored document. ---! @param b {{prefix}}.query_json_tree The needle. +--! @param a public.{{prefix}}_json_search The stored document. +--! @param b {{prefix}}.query_json The needle. --! @return boolean True when `b` contains `a`. -CREATE FUNCTION {{prefix}}.json_tree_contained_by(a public.{{prefix}}_json_tree, b {{prefix}}.query_json_tree) +CREATE FUNCTION {{prefix}}.json_tree_contained_by(a public.{{prefix}}_json_search, b {{prefix}}.query_json) RETURNS boolean LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE AS $$ @@ -440,10 +440,10 @@ AS $$ $$; --! @brief Does the stored document `a` contain the stored document `b`? ---! @param a public.{{prefix}}_json_tree The container. ---! @param b public.{{prefix}}_json_tree The contained document. +--! @param a public.{{prefix}}_json_search The container. +--! @param b public.{{prefix}}_json_search The contained document. --! @return boolean True when `a`'s projection contains `b`'s. -CREATE FUNCTION {{prefix}}.json_tree_document_contains(a public.{{prefix}}_json_tree, b public.{{prefix}}_json_tree) +CREATE FUNCTION {{prefix}}.json_tree_document_contains(a public.{{prefix}}_json_search, b public.{{prefix}}_json_search) RETURNS boolean LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE AS $$ @@ -524,15 +524,15 @@ $$; --! opening on its own. Enumerating one level needs no child's token, so the --! rows are the same whichever rule each child's token takes. --! ---! @param e public.{{prefix}}_json_tree The stored document. +--! @param e public.{{prefix}}_json_search The stored document. --! @param token text The node's token, base64: derived by the client from --! the node's path alone, and never stored. ---! @return SETOF public.{{prefix}}_json_tree_entry The children, each with +--! @return SETOF public.{{prefix}}_json_entry The children, each with --! `d = 1`; no rows for a token that names no node, for a scalar or --! an empty container, and for the root of a document whose root is --! sealed, whose top-level fields chain from nothing. -CREATE FUNCTION {{prefix}}.json_tree_children(e public.{{prefix}}_json_tree, token text) - RETURNS SETOF public.{{prefix}}_json_tree_entry +CREATE FUNCTION {{prefix}}.json_tree_children(e public.{{prefix}}_json_search, token text) + RETURNS SETOF public.{{prefix}}_json_entry LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE AS $$ WITH RECURSIVE @@ -564,7 +564,7 @@ AS $$ visited AS MATERIALIZED ( SELECT p.* FROM probe AS p LIMIT (SELECT doc.entries + 1 FROM doc) ) - SELECT (doc.meta || (doc.sv -> p.j) || jsonb_build_object('j', p.j, 'd', 1))::public.{{prefix}}_json_tree_entry + SELECT (doc.meta || (doc.sv -> p.j) || jsonb_build_object('j', p.j, 'd', 1))::public.{{prefix}}_json_entry FROM visited AS p, doc WHERE p.j IS NOT NULL ORDER BY p.j @@ -577,14 +577,14 @@ $$; --! built once, where calling that function after finding the node would --! build it twice. --! ---! @param e public.{{prefix}}_json_tree The stored document. +--! @param e public.{{prefix}}_json_search The stored document. --! @param token text The node's token, base64. --! @return integer The count; `0` for a scalar or an empty container; NULL --! for a token that names no node, as `jsonb_array_length` of a --! missing path is, and for the root of a document whose root is --! sealed, whose top-level fields chain from nothing and so cannot --! be counted from its token. -CREATE FUNCTION {{prefix}}.json_tree_child_count(e public.{{prefix}}_json_tree, token text) +CREATE FUNCTION {{prefix}}.json_tree_child_count(e public.{{prefix}}_json_search, token text) RETURNS integer LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE AS $$ @@ -659,16 +659,16 @@ $$; --! is not sealed is always read in full, since its root holds no document --! to return. --! ---! @param e public.{{prefix}}_json_tree The stored document. +--! @param e public.{{prefix}}_json_search The stored document. --! @param tokens text[] The tokens of the nodes to read, base64, each --! derived by the client from a concrete path alone. --! @param budget integer The most rows to return before falling back to --! the sealed root; `0` to always read in full. Default 64. ---! @return SETOF public.{{prefix}}_json_tree_entry The rows, each with its +--! @return SETOF public.{{prefix}}_json_entry The rows, each with its --! depth `d` below the node its token named; or the root entry --! alone, with `j = 0` and `d = 0`, past the budget. -CREATE FUNCTION {{prefix}}.json_tree_subtree(e public.{{prefix}}_json_tree, tokens text[], budget integer DEFAULT 64) - RETURNS SETOF public.{{prefix}}_json_tree_entry +CREATE FUNCTION {{prefix}}.json_tree_subtree(e public.{{prefix}}_json_search, tokens text[], budget integer DEFAULT 64) + RETURNS SETOF public.{{prefix}}_json_entry LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE AS $$ WITH RECURSIVE @@ -750,12 +750,12 @@ AS $$ SELECT rows.entry FROM ( SELECT 0::bigint AS k, 0 AS j, - (doc.meta || (doc.sv -> 0) || '{"j": 0, "d": 0}'::jsonb)::public.{{prefix}}_json_tree_entry AS entry + (doc.meta || (doc.sv -> 0) || '{"j": 0, "d": 0}'::jsonb)::public.{{prefix}}_json_entry AS entry FROM doc, over WHERE over.fallback UNION ALL SELECT f.k, f.j, - (doc.meta || f.entry || jsonb_build_object('j', f.j, 'd', f.depth))::public.{{prefix}}_json_tree_entry + (doc.meta || f.entry || jsonb_build_object('j', f.j, 'd', f.depth))::public.{{prefix}}_json_entry FROM found AS f, doc, over WHERE NOT over.fallback ) AS rows @@ -767,9 +767,9 @@ $$; ------------------------------------------------------------------------------ --! @brief The path selector of an entry. ---! @param entry public.{{prefix}}_json_tree_entry The entry. +--! @param entry public.{{prefix}}_json_entry The entry. --! @return text Its `s`, base64. -CREATE FUNCTION {{prefix}}.selector(entry public.{{prefix}}_json_tree_entry) +CREATE FUNCTION {{prefix}}.selector(entry public.{{prefix}}_json_entry) RETURNS text LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE AS $$ @@ -790,9 +790,9 @@ $$; --! the entry object, which sorts by ciphertext, and an operator class that --! would redirect it needs a superuser to install. --! ---! @param entry public.{{prefix}}_json_tree_entry The entry. +--! @param entry public.{{prefix}}_json_entry The entry. --! @return {{prefix}}_internal.ope_cllw The term, or NULL. -CREATE FUNCTION {{prefix}}.ord_term(entry public.{{prefix}}_json_tree_entry) +CREATE FUNCTION {{prefix}}.ord_term(entry public.{{prefix}}_json_entry) RETURNS {{prefix}}_internal.ope_cllw LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE AS $$ @@ -800,9 +800,9 @@ AS $$ $$; --! @brief The leaf ordering term of an ordering operand. ---! @param q {{prefix}}.query_json_tree_leaf The operand. +--! @param q {{prefix}}.query_json_leaf The operand. --! @return {{prefix}}_internal.ope_cllw Its `op`, base64-decoded. -CREATE FUNCTION {{prefix}}.ord_term(q {{prefix}}.query_json_tree_leaf) +CREATE FUNCTION {{prefix}}.ord_term(q {{prefix}}.query_json_leaf) RETURNS {{prefix}}_internal.ope_cllw LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE AS $$ @@ -811,9 +811,9 @@ $$; --! @brief The path selector of an ordering operand: what `->` takes to reach --! the leaf the operand compares against. ---! @param q {{prefix}}.query_json_tree_leaf The operand. +--! @param q {{prefix}}.query_json_leaf The operand. --! @return text Its `s`, base64. -CREATE FUNCTION {{prefix}}.selector(q {{prefix}}.query_json_tree_leaf) +CREATE FUNCTION {{prefix}}.selector(q {{prefix}}.query_json_leaf) RETURNS text LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE AS $$ diff --git a/packages/eql/src/v3/json_tree/operators.sql b/packages/eql/src/v3/json_tree/operators.sql index fbb30e1e4..3ea6a8980 100644 --- a/packages/eql/src/v3/json_tree/operators.sql +++ b/packages/eql/src/v3/json_tree/operators.sql @@ -5,7 +5,7 @@ -- REQUIRE: src/v3/sem/ope_cllw/types.sql --! @file v3/json_tree/operators.sql ---! @brief Operators on public.{{prefix}}_json_tree and public.{{prefix}}_json_tree_entry. +--! @brief Operators on public.{{prefix}}_json_search and public.{{prefix}}_json_entry. ------------------------------------------------------------------------------ -- -> and ->> accessors (by selector) @@ -23,11 +23,11 @@ --! literal resolves to the NATIVE `jsonb -> text` operator and silently --! returns native jsonb semantics, as it does on every EQL jsonb domain. --! ---! @param e public.{{prefix}}_json_tree The stored document. +--! @param e public.{{prefix}}_json_search The stored document. --! @param selector text The path selector, base64. ---! @return public.{{prefix}}_json_tree_entry The entry, or NULL. -CREATE FUNCTION {{prefix}}."->"(e public.{{prefix}}_json_tree, selector text) - RETURNS public.{{prefix}}_json_tree_entry +--! @return public.{{prefix}}_json_entry The entry, or NULL. +CREATE FUNCTION {{prefix}}."->"(e public.{{prefix}}_json_search, selector text) + RETURNS public.{{prefix}}_json_entry LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE AS $$ SELECT {{prefix}}.json_tree_path_query_first(e::jsonb, selector) @@ -35,16 +35,16 @@ $$; CREATE OPERATOR ->( FUNCTION={{prefix}}."->", - LEFTARG=public.{{prefix}}_json_tree, + LEFTARG=public.{{prefix}}_json_search, RIGHTARG=text ); --! @brief ->> operator with a text selector: the first entry at the path, as --! JSON text. It does not decrypt. ---! @param e public.{{prefix}}_json_tree The stored document. +--! @param e public.{{prefix}}_json_search The stored document. --! @param selector text The path selector, base64. --! @return text The entry as text, or NULL. -CREATE FUNCTION {{prefix}}."->>"(e public.{{prefix}}_json_tree, selector text) +CREATE FUNCTION {{prefix}}."->>"(e public.{{prefix}}_json_search, selector text) RETURNS text LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE AS $$ @@ -53,7 +53,7 @@ $$; CREATE OPERATOR ->> ( FUNCTION={{prefix}}."->>", - LEFTARG=public.{{prefix}}_json_tree, + LEFTARG=public.{{prefix}}_json_search, RIGHTARG=text ); @@ -61,18 +61,18 @@ CREATE OPERATOR ->> ( -- @> and <@ containment ------------------------------------------------------------------------------ ---! @brief @> contains operator with a query_json_tree needle. +--! @brief @> contains operator with a query_json needle. --! --! Inlines to native `jsonb @>` over `{{prefix}}.json_tree_query(a)`, so --! a functional GIN index on that expression engages. Not STRICT, for the --! reason `{{prefix}}.json_tree_contains` gives: a STRICT wrapper around a --! non-STRICT body is not inlined. --! ---! @param a public.{{prefix}}_json_tree The container. ---! @param b {{prefix}}.query_json_tree The needle. +--! @param a public.{{prefix}}_json_search The container. +--! @param b {{prefix}}.query_json The needle. --! @return boolean True when `a` contains `b`; NULL when either is NULL. --! @see {{prefix}}.json_tree_contains -CREATE FUNCTION {{prefix}}."@>"(a public.{{prefix}}_json_tree, b {{prefix}}.query_json_tree) +CREATE FUNCTION {{prefix}}."@>"(a public.{{prefix}}_json_search, b {{prefix}}.query_json) RETURNS boolean LANGUAGE sql IMMUTABLE PARALLEL SAFE AS $$ @@ -81,15 +81,15 @@ $$; CREATE OPERATOR @>( FUNCTION={{prefix}}."@>", - LEFTARG=public.{{prefix}}_json_tree, - RIGHTARG={{prefix}}.query_json_tree + LEFTARG=public.{{prefix}}_json_search, + RIGHTARG={{prefix}}.query_json ); --! @brief @> contains operator (document, document). ---! @param a public.{{prefix}}_json_tree The container. ---! @param b public.{{prefix}}_json_tree The contained document. +--! @param a public.{{prefix}}_json_search The container. +--! @param b public.{{prefix}}_json_search The contained document. --! @return boolean True when `a` contains `b`. -CREATE FUNCTION {{prefix}}."@>"(a public.{{prefix}}_json_tree, b public.{{prefix}}_json_tree) +CREATE FUNCTION {{prefix}}."@>"(a public.{{prefix}}_json_search, b public.{{prefix}}_json_search) RETURNS boolean LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE AS $$ @@ -98,16 +98,16 @@ $$; CREATE OPERATOR @>( FUNCTION={{prefix}}."@>", - LEFTARG=public.{{prefix}}_json_tree, - RIGHTARG=public.{{prefix}}_json_tree + LEFTARG=public.{{prefix}}_json_search, + RIGHTARG=public.{{prefix}}_json_search ); ---! @brief <@ contained-by operator with a query_json_tree container. ---! @param a public.{{prefix}}_json_tree The contained document. ---! @param b {{prefix}}.query_json_tree The needle that must contain it. +--! @brief <@ contained-by operator with a query_json container. +--! @param a public.{{prefix}}_json_search The contained document. +--! @param b {{prefix}}.query_json The needle that must contain it. --! @return boolean True when `b` contains `a`. --! @see {{prefix}}.json_tree_contained_by -CREATE FUNCTION {{prefix}}."<@"(a public.{{prefix}}_json_tree, b {{prefix}}.query_json_tree) +CREATE FUNCTION {{prefix}}."<@"(a public.{{prefix}}_json_search, b {{prefix}}.query_json) RETURNS boolean LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE AS $$ @@ -116,16 +116,16 @@ $$; CREATE OPERATOR <@( FUNCTION={{prefix}}."<@", - LEFTARG=public.{{prefix}}_json_tree, - RIGHTARG={{prefix}}.query_json_tree + LEFTARG=public.{{prefix}}_json_search, + RIGHTARG={{prefix}}.query_json ); ---! @brief <@ contained-by operator with a query_json_tree LHS: `b @> a`. +--! @brief <@ contained-by operator with a query_json LHS: `b @> a`. --! Not STRICT, as `@>` is not. ---! @param a {{prefix}}.query_json_tree The needle. ---! @param b public.{{prefix}}_json_tree The container. +--! @param a {{prefix}}.query_json The needle. +--! @param b public.{{prefix}}_json_search The container. --! @return boolean True when `b` contains `a`; NULL when either is NULL. -CREATE FUNCTION {{prefix}}."<@"(a {{prefix}}.query_json_tree, b public.{{prefix}}_json_tree) +CREATE FUNCTION {{prefix}}."<@"(a {{prefix}}.query_json, b public.{{prefix}}_json_search) RETURNS boolean LANGUAGE sql IMMUTABLE PARALLEL SAFE AS $$ @@ -134,15 +134,15 @@ $$; CREATE OPERATOR <@( FUNCTION={{prefix}}."<@", - LEFTARG={{prefix}}.query_json_tree, - RIGHTARG=public.{{prefix}}_json_tree + LEFTARG={{prefix}}.query_json, + RIGHTARG=public.{{prefix}}_json_search ); --! @brief <@ contained-by operator (document, document). ---! @param a public.{{prefix}}_json_tree The contained document. ---! @param b public.{{prefix}}_json_tree The container. +--! @param a public.{{prefix}}_json_search The contained document. +--! @param b public.{{prefix}}_json_search The container. --! @return boolean True when `b` contains `a`. -CREATE FUNCTION {{prefix}}."<@"(a public.{{prefix}}_json_tree, b public.{{prefix}}_json_tree) +CREATE FUNCTION {{prefix}}."<@"(a public.{{prefix}}_json_search, b public.{{prefix}}_json_search) RETURNS boolean LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE AS $$ @@ -151,8 +151,8 @@ $$; CREATE OPERATOR <@( FUNCTION={{prefix}}."<@", - LEFTARG=public.{{prefix}}_json_tree, - RIGHTARG=public.{{prefix}}_json_tree + LEFTARG=public.{{prefix}}_json_search, + RIGHTARG=public.{{prefix}}_json_search ); ------------------------------------------------------------------------------ @@ -161,10 +161,10 @@ CREATE OPERATOR <@( --! @brief Less-than on an extracted entry against an ordering operand, via --! the CLLW OPE term (native bytea order). ---! @param a public.{{prefix}}_json_tree_entry The entry, `col -> 'sel'`. ---! @param b {{prefix}}.query_json_tree_leaf The operand. +--! @param a public.{{prefix}}_json_entry The entry, `col -> 'sel'`. +--! @param b {{prefix}}.query_json_leaf The operand. --! @return boolean True when the entry's leaf is below the operand's. -CREATE FUNCTION {{prefix}}.lt(a public.{{prefix}}_json_tree_entry, b {{prefix}}.query_json_tree_leaf) +CREATE FUNCTION {{prefix}}.lt(a public.{{prefix}}_json_entry, b {{prefix}}.query_json_leaf) RETURNS boolean LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE AS $$ @@ -173,18 +173,18 @@ $$; CREATE OPERATOR < ( FUNCTION = {{prefix}}.lt, - LEFTARG = public.{{prefix}}_json_tree_entry, - RIGHTARG = {{prefix}}.query_json_tree_leaf, + LEFTARG = public.{{prefix}}_json_entry, + RIGHTARG = {{prefix}}.query_json_leaf, NEGATOR = >=, RESTRICT = scalarltsel, JOIN = scalarltjoinsel ); --! @brief Less-than-or-equal on an extracted entry against an ordering operand. ---! @param a public.{{prefix}}_json_tree_entry The entry. ---! @param b {{prefix}}.query_json_tree_leaf The operand. +--! @param a public.{{prefix}}_json_entry The entry. +--! @param b {{prefix}}.query_json_leaf The operand. --! @return boolean True when the entry's leaf is at or below the operand's. -CREATE FUNCTION {{prefix}}.lte(a public.{{prefix}}_json_tree_entry, b {{prefix}}.query_json_tree_leaf) +CREATE FUNCTION {{prefix}}.lte(a public.{{prefix}}_json_entry, b {{prefix}}.query_json_leaf) RETURNS boolean LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE AS $$ @@ -193,18 +193,18 @@ $$; CREATE OPERATOR <= ( FUNCTION = {{prefix}}.lte, - LEFTARG = public.{{prefix}}_json_tree_entry, - RIGHTARG = {{prefix}}.query_json_tree_leaf, + LEFTARG = public.{{prefix}}_json_entry, + RIGHTARG = {{prefix}}.query_json_leaf, NEGATOR = >, RESTRICT = scalarlesel, JOIN = scalarlejoinsel ); --! @brief Greater-than on an extracted entry against an ordering operand. ---! @param a public.{{prefix}}_json_tree_entry The entry. ---! @param b {{prefix}}.query_json_tree_leaf The operand. +--! @param a public.{{prefix}}_json_entry The entry. +--! @param b {{prefix}}.query_json_leaf The operand. --! @return boolean True when the entry's leaf is above the operand's. -CREATE FUNCTION {{prefix}}.gt(a public.{{prefix}}_json_tree_entry, b {{prefix}}.query_json_tree_leaf) +CREATE FUNCTION {{prefix}}.gt(a public.{{prefix}}_json_entry, b {{prefix}}.query_json_leaf) RETURNS boolean LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE AS $$ @@ -213,18 +213,18 @@ $$; CREATE OPERATOR > ( FUNCTION = {{prefix}}.gt, - LEFTARG = public.{{prefix}}_json_tree_entry, - RIGHTARG = {{prefix}}.query_json_tree_leaf, + LEFTARG = public.{{prefix}}_json_entry, + RIGHTARG = {{prefix}}.query_json_leaf, NEGATOR = <=, RESTRICT = scalargtsel, JOIN = scalargtjoinsel ); --! @brief Greater-than-or-equal on an extracted entry against an ordering operand. ---! @param a public.{{prefix}}_json_tree_entry The entry. ---! @param b {{prefix}}.query_json_tree_leaf The operand. +--! @param a public.{{prefix}}_json_entry The entry. +--! @param b {{prefix}}.query_json_leaf The operand. --! @return boolean True when the entry's leaf is at or above the operand's. -CREATE FUNCTION {{prefix}}.gte(a public.{{prefix}}_json_tree_entry, b {{prefix}}.query_json_tree_leaf) +CREATE FUNCTION {{prefix}}.gte(a public.{{prefix}}_json_entry, b {{prefix}}.query_json_leaf) RETURNS boolean LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE AS $$ @@ -233,8 +233,8 @@ $$; CREATE OPERATOR >= ( FUNCTION = {{prefix}}.gte, - LEFTARG = public.{{prefix}}_json_tree_entry, - RIGHTARG = {{prefix}}.query_json_tree_leaf, + LEFTARG = public.{{prefix}}_json_entry, + RIGHTARG = {{prefix}}.query_json_leaf, NEGATOR = <, RESTRICT = scalargesel, JOIN = scalargejoinsel @@ -245,10 +245,10 @@ CREATE OPERATOR >= ( ------------------------------------------------------------------------------ --! @brief Less-than between two extracted entries via their OPE terms. ---! @param a public.{{prefix}}_json_tree_entry Left operand. ---! @param b public.{{prefix}}_json_tree_entry Right operand. +--! @param a public.{{prefix}}_json_entry Left operand. +--! @param b public.{{prefix}}_json_entry Right operand. --! @return boolean True when `a`'s leaf is below `b`'s. -CREATE FUNCTION {{prefix}}.lt(a public.{{prefix}}_json_tree_entry, b public.{{prefix}}_json_tree_entry) +CREATE FUNCTION {{prefix}}.lt(a public.{{prefix}}_json_entry, b public.{{prefix}}_json_entry) RETURNS boolean LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE AS $$ @@ -257,8 +257,8 @@ $$; CREATE OPERATOR < ( FUNCTION = {{prefix}}.lt, - LEFTARG = public.{{prefix}}_json_tree_entry, - RIGHTARG = public.{{prefix}}_json_tree_entry, + LEFTARG = public.{{prefix}}_json_entry, + RIGHTARG = public.{{prefix}}_json_entry, COMMUTATOR = >, NEGATOR = >=, RESTRICT = scalarltsel, @@ -266,10 +266,10 @@ CREATE OPERATOR < ( ); --! @brief Less-than-or-equal between two extracted entries. ---! @param a public.{{prefix}}_json_tree_entry Left operand. ---! @param b public.{{prefix}}_json_tree_entry Right operand. +--! @param a public.{{prefix}}_json_entry Left operand. +--! @param b public.{{prefix}}_json_entry Right operand. --! @return boolean True when `a`'s leaf is at or below `b`'s. -CREATE FUNCTION {{prefix}}.lte(a public.{{prefix}}_json_tree_entry, b public.{{prefix}}_json_tree_entry) +CREATE FUNCTION {{prefix}}.lte(a public.{{prefix}}_json_entry, b public.{{prefix}}_json_entry) RETURNS boolean LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE AS $$ @@ -278,8 +278,8 @@ $$; CREATE OPERATOR <= ( FUNCTION = {{prefix}}.lte, - LEFTARG = public.{{prefix}}_json_tree_entry, - RIGHTARG = public.{{prefix}}_json_tree_entry, + LEFTARG = public.{{prefix}}_json_entry, + RIGHTARG = public.{{prefix}}_json_entry, COMMUTATOR = >=, NEGATOR = >, RESTRICT = scalarlesel, @@ -287,10 +287,10 @@ CREATE OPERATOR <= ( ); --! @brief Greater-than between two extracted entries. ---! @param a public.{{prefix}}_json_tree_entry Left operand. ---! @param b public.{{prefix}}_json_tree_entry Right operand. +--! @param a public.{{prefix}}_json_entry Left operand. +--! @param b public.{{prefix}}_json_entry Right operand. --! @return boolean True when `a`'s leaf is above `b`'s. -CREATE FUNCTION {{prefix}}.gt(a public.{{prefix}}_json_tree_entry, b public.{{prefix}}_json_tree_entry) +CREATE FUNCTION {{prefix}}.gt(a public.{{prefix}}_json_entry, b public.{{prefix}}_json_entry) RETURNS boolean LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE AS $$ @@ -299,8 +299,8 @@ $$; CREATE OPERATOR > ( FUNCTION = {{prefix}}.gt, - LEFTARG = public.{{prefix}}_json_tree_entry, - RIGHTARG = public.{{prefix}}_json_tree_entry, + LEFTARG = public.{{prefix}}_json_entry, + RIGHTARG = public.{{prefix}}_json_entry, COMMUTATOR = <, NEGATOR = <=, RESTRICT = scalargtsel, @@ -308,10 +308,10 @@ CREATE OPERATOR > ( ); --! @brief Greater-than-or-equal between two extracted entries. ---! @param a public.{{prefix}}_json_tree_entry Left operand. ---! @param b public.{{prefix}}_json_tree_entry Right operand. +--! @param a public.{{prefix}}_json_entry Left operand. +--! @param b public.{{prefix}}_json_entry Right operand. --! @return boolean True when `a`'s leaf is at or above `b`'s. -CREATE FUNCTION {{prefix}}.gte(a public.{{prefix}}_json_tree_entry, b public.{{prefix}}_json_tree_entry) +CREATE FUNCTION {{prefix}}.gte(a public.{{prefix}}_json_entry, b public.{{prefix}}_json_entry) RETURNS boolean LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE AS $$ @@ -320,8 +320,8 @@ $$; CREATE OPERATOR >= ( FUNCTION = {{prefix}}.gte, - LEFTARG = public.{{prefix}}_json_tree_entry, - RIGHTARG = public.{{prefix}}_json_tree_entry, + LEFTARG = public.{{prefix}}_json_entry, + RIGHTARG = public.{{prefix}}_json_entry, COMMUTATOR = <=, NEGATOR = <, RESTRICT = scalargesel, diff --git a/packages/eql/src/v3/json_tree/types.sql b/packages/eql/src/v3/json_tree/types.sql index 52ad558aa..536c3e9b8 100644 --- a/packages/eql/src/v3/json_tree/types.sql +++ b/packages/eql/src/v3/json_tree/types.sql @@ -10,12 +10,12 @@ --! Stack Encrypt's JSON index writes. It shares no wire shape and no SQL with --! the SteVec surface. Four jsonb-backed domains (none over another domain, --! so the native-jsonb firewall in blockers.sql attaches): ---! - public.{{prefix}}_json_tree — storage: an EQL envelope `{v, i, h, sv, q}`. ---! - public.{{prefix}}_json_tree_entry — one `sv` element (returned by `->` +--! - public.{{prefix}}_json_search — storage: an EQL envelope `{v, i, h, sv, q}`. +--! - public.{{prefix}}_json_entry — one `sv` element (returned by `->` --! and the subtree read). ---! - {{prefix}}.query_json_tree — a containment needle: the element-nested +--! - {{prefix}}.query_json — a containment needle: the element-nested --! projection of the needle's selectors. ---! - {{prefix}}.query_json_tree_leaf — an ordering operand at a path: a +--! - {{prefix}}.query_json_leaf — an ordering operand at a path: a --! selector and a leaf ordering term. --! --! The design is `docs/plans/2026-10-17-json-tree-index.md` @@ -184,20 +184,20 @@ $$; --! reaching a column value. --! --! @note Constructing from inline JSON uses the standard DOMAIN cast: ---! `'{"v":{{eql_version}},"i":{…},"h":"…","sv":[…],"q":{…}}'::public.{{prefix}}_json_tree`. +--! `'{"v":{{eql_version}},"i":{…},"h":"…","sv":[…],"q":{…}}'::public.{{prefix}}_json_search`. DO $$ BEGIN IF NOT EXISTS ( SELECT 1 FROM pg_type - WHERE typname = '{{prefix}}_json_tree' AND typnamespace = 'public'::regnamespace + WHERE typname = '{{prefix}}_json_search' AND typnamespace = 'public'::regnamespace ) THEN - CREATE DOMAIN public.{{prefix}}_json_tree AS jsonb + CREATE DOMAIN public.{{prefix}}_json_search AS jsonb CHECK ( public.{{prefix}}_is_valid_json_tree_document_payload(VALUE) ); END IF; - COMMENT ON DOMAIN public.{{prefix}}_json_tree IS 'EQL encrypted JSON tree document (containment, token chain)'; + COMMENT ON DOMAIN public.{{prefix}}_json_search IS 'EQL encrypted JSON tree document (containment, token chain)'; END $$; @@ -214,15 +214,15 @@ DO $$ BEGIN IF NOT EXISTS ( SELECT 1 FROM pg_type - WHERE typname = '{{prefix}}_json_tree_entry' AND typnamespace = 'public'::regnamespace + WHERE typname = '{{prefix}}_json_entry' AND typnamespace = 'public'::regnamespace ) THEN - CREATE DOMAIN public.{{prefix}}_json_tree_entry AS jsonb + CREATE DOMAIN public.{{prefix}}_json_entry AS jsonb CHECK ( VALUE IS NULL OR public.{{prefix}}_is_valid_json_tree_entry_payload(VALUE) ); END IF; - COMMENT ON DOMAIN public.{{prefix}}_json_tree_entry IS 'EQL encrypted JSON tree entry (one node)'; + COMMENT ON DOMAIN public.{{prefix}}_json_entry IS 'EQL encrypted JSON tree entry (one node)'; END $$; @@ -235,21 +235,21 @@ $$; --! from casting and matching every row via bare `jsonb @>`. --! --! @note Construct from inline JSON via the DOMAIN cast: ---! `'{"sv":[…],"el":[]}'::{{prefix}}.query_json_tree`. +--! `'{"sv":[…],"el":[]}'::{{prefix}}.query_json`. --! @see {{prefix}}.json_tree_query DO $$ BEGIN IF NOT EXISTS ( SELECT 1 FROM pg_type - WHERE typname = 'query_json_tree' AND typnamespace = '{{prefix}}'::regnamespace + WHERE typname = 'query_json' AND typnamespace = '{{prefix}}'::regnamespace ) THEN - CREATE DOMAIN {{prefix}}.query_json_tree AS jsonb + CREATE DOMAIN {{prefix}}.query_json AS jsonb CHECK ( public.{{prefix}}_is_valid_json_tree_query_payload(VALUE) ); END IF; - COMMENT ON DOMAIN {{prefix}}.query_json_tree IS 'EQL JSON tree query operand (containment)'; + COMMENT ON DOMAIN {{prefix}}.query_json IS 'EQL JSON tree query operand (containment)'; END $$; @@ -261,14 +261,14 @@ $$; --! `>` and `>=` on an extracted entry; a range at a path is two predicates. --! --! @note Construct from inline JSON via the DOMAIN cast: ---! `'{"s":"…","op":"…"}'::{{prefix}}.query_json_tree_leaf`. +--! `'{"s":"…","op":"…"}'::{{prefix}}.query_json_leaf`. DO $$ BEGIN IF NOT EXISTS ( SELECT 1 FROM pg_type - WHERE typname = 'query_json_tree_leaf' AND typnamespace = '{{prefix}}'::regnamespace + WHERE typname = 'query_json_leaf' AND typnamespace = '{{prefix}}'::regnamespace ) THEN - CREATE DOMAIN {{prefix}}.query_json_tree_leaf AS jsonb + CREATE DOMAIN {{prefix}}.query_json_leaf AS jsonb CHECK ( VALUE IS NULL OR COALESCE( @@ -281,6 +281,6 @@ BEGIN ); END IF; - COMMENT ON DOMAIN {{prefix}}.query_json_tree_leaf IS 'EQL JSON tree query operand (ordering at a path)'; + COMMENT ON DOMAIN {{prefix}}.query_json_leaf IS 'EQL JSON tree query operand (ordering at a path)'; END $$; diff --git a/packages/eql/src/v3/scalars/bigint/json_entry_bigint_functions.sql b/packages/eql/src/v3/scalars/bigint/json_entry_bigint_functions.sql index da47e1897..2098bf830 100644 --- a/packages/eql/src/v3/scalars/bigint/json_entry_bigint_functions.sql +++ b/packages/eql/src/v3/scalars/bigint/json_entry_bigint_functions.sql @@ -1,4 +1,5 @@ -- AUTOMATICALLY GENERATED FILE. +-- ONLY EQL VERSION: 3 -- REQUIRE: src/v3/schema.sql -- REQUIRE: src/v3/json/types.sql -- REQUIRE: src/v3/json/functions.sql diff --git a/packages/eql/src/v3/scalars/bigint/json_entry_bigint_operators.sql b/packages/eql/src/v3/scalars/bigint/json_entry_bigint_operators.sql index 3403a3c1f..30a2ce039 100644 --- a/packages/eql/src/v3/scalars/bigint/json_entry_bigint_operators.sql +++ b/packages/eql/src/v3/scalars/bigint/json_entry_bigint_operators.sql @@ -1,4 +1,5 @@ -- AUTOMATICALLY GENERATED FILE. +-- ONLY EQL VERSION: 3 -- REQUIRE: src/v3/schema.sql -- REQUIRE: src/v3/json/types.sql -- REQUIRE: src/v3/scalars/bigint/query_bigint_types.sql diff --git a/packages/eql/src/v3/scalars/date/json_entry_date_functions.sql b/packages/eql/src/v3/scalars/date/json_entry_date_functions.sql index 9e9f91635..7590fc9ab 100644 --- a/packages/eql/src/v3/scalars/date/json_entry_date_functions.sql +++ b/packages/eql/src/v3/scalars/date/json_entry_date_functions.sql @@ -1,4 +1,5 @@ -- AUTOMATICALLY GENERATED FILE. +-- ONLY EQL VERSION: 3 -- REQUIRE: src/v3/schema.sql -- REQUIRE: src/v3/json/types.sql -- REQUIRE: src/v3/json/functions.sql diff --git a/packages/eql/src/v3/scalars/date/json_entry_date_operators.sql b/packages/eql/src/v3/scalars/date/json_entry_date_operators.sql index 70cbcf9c8..14da36ebd 100644 --- a/packages/eql/src/v3/scalars/date/json_entry_date_operators.sql +++ b/packages/eql/src/v3/scalars/date/json_entry_date_operators.sql @@ -1,4 +1,5 @@ -- AUTOMATICALLY GENERATED FILE. +-- ONLY EQL VERSION: 3 -- REQUIRE: src/v3/schema.sql -- REQUIRE: src/v3/json/types.sql -- REQUIRE: src/v3/scalars/date/query_date_types.sql diff --git a/packages/eql/src/v3/scalars/double/json_entry_double_functions.sql b/packages/eql/src/v3/scalars/double/json_entry_double_functions.sql index ef2171dd9..03159abde 100644 --- a/packages/eql/src/v3/scalars/double/json_entry_double_functions.sql +++ b/packages/eql/src/v3/scalars/double/json_entry_double_functions.sql @@ -1,4 +1,5 @@ -- AUTOMATICALLY GENERATED FILE. +-- ONLY EQL VERSION: 3 -- REQUIRE: src/v3/schema.sql -- REQUIRE: src/v3/json/types.sql -- REQUIRE: src/v3/json/functions.sql diff --git a/packages/eql/src/v3/scalars/double/json_entry_double_operators.sql b/packages/eql/src/v3/scalars/double/json_entry_double_operators.sql index 391e2743e..cbefec5ec 100644 --- a/packages/eql/src/v3/scalars/double/json_entry_double_operators.sql +++ b/packages/eql/src/v3/scalars/double/json_entry_double_operators.sql @@ -1,4 +1,5 @@ -- AUTOMATICALLY GENERATED FILE. +-- ONLY EQL VERSION: 3 -- REQUIRE: src/v3/schema.sql -- REQUIRE: src/v3/json/types.sql -- REQUIRE: src/v3/scalars/double/query_double_types.sql diff --git a/packages/eql/src/v3/scalars/integer/json_entry_integer_functions.sql b/packages/eql/src/v3/scalars/integer/json_entry_integer_functions.sql index b46db4e60..5aec51640 100644 --- a/packages/eql/src/v3/scalars/integer/json_entry_integer_functions.sql +++ b/packages/eql/src/v3/scalars/integer/json_entry_integer_functions.sql @@ -1,4 +1,5 @@ -- AUTOMATICALLY GENERATED FILE. +-- ONLY EQL VERSION: 3 -- REQUIRE: src/v3/schema.sql -- REQUIRE: src/v3/json/types.sql -- REQUIRE: src/v3/json/functions.sql diff --git a/packages/eql/src/v3/scalars/integer/json_entry_integer_operators.sql b/packages/eql/src/v3/scalars/integer/json_entry_integer_operators.sql index f58a9ca2c..f28018e07 100644 --- a/packages/eql/src/v3/scalars/integer/json_entry_integer_operators.sql +++ b/packages/eql/src/v3/scalars/integer/json_entry_integer_operators.sql @@ -1,4 +1,5 @@ -- AUTOMATICALLY GENERATED FILE. +-- ONLY EQL VERSION: 3 -- REQUIRE: src/v3/schema.sql -- REQUIRE: src/v3/json/types.sql -- REQUIRE: src/v3/scalars/integer/query_integer_types.sql diff --git a/packages/eql/src/v3/scalars/numeric/json_entry_numeric_functions.sql b/packages/eql/src/v3/scalars/numeric/json_entry_numeric_functions.sql index 4d32069a1..d5110a065 100644 --- a/packages/eql/src/v3/scalars/numeric/json_entry_numeric_functions.sql +++ b/packages/eql/src/v3/scalars/numeric/json_entry_numeric_functions.sql @@ -1,4 +1,5 @@ -- AUTOMATICALLY GENERATED FILE. +-- ONLY EQL VERSION: 3 -- REQUIRE: src/v3/schema.sql -- REQUIRE: src/v3/json/types.sql -- REQUIRE: src/v3/json/functions.sql diff --git a/packages/eql/src/v3/scalars/numeric/json_entry_numeric_operators.sql b/packages/eql/src/v3/scalars/numeric/json_entry_numeric_operators.sql index 49aad090b..25b425d83 100644 --- a/packages/eql/src/v3/scalars/numeric/json_entry_numeric_operators.sql +++ b/packages/eql/src/v3/scalars/numeric/json_entry_numeric_operators.sql @@ -1,4 +1,5 @@ -- AUTOMATICALLY GENERATED FILE. +-- ONLY EQL VERSION: 3 -- REQUIRE: src/v3/schema.sql -- REQUIRE: src/v3/json/types.sql -- REQUIRE: src/v3/scalars/numeric/query_numeric_types.sql diff --git a/packages/eql/src/v3/scalars/real/json_entry_real_functions.sql b/packages/eql/src/v3/scalars/real/json_entry_real_functions.sql index 6cab30901..47803945f 100644 --- a/packages/eql/src/v3/scalars/real/json_entry_real_functions.sql +++ b/packages/eql/src/v3/scalars/real/json_entry_real_functions.sql @@ -1,4 +1,5 @@ -- AUTOMATICALLY GENERATED FILE. +-- ONLY EQL VERSION: 3 -- REQUIRE: src/v3/schema.sql -- REQUIRE: src/v3/json/types.sql -- REQUIRE: src/v3/json/functions.sql diff --git a/packages/eql/src/v3/scalars/real/json_entry_real_operators.sql b/packages/eql/src/v3/scalars/real/json_entry_real_operators.sql index 4ddcc8522..26b26a117 100644 --- a/packages/eql/src/v3/scalars/real/json_entry_real_operators.sql +++ b/packages/eql/src/v3/scalars/real/json_entry_real_operators.sql @@ -1,4 +1,5 @@ -- AUTOMATICALLY GENERATED FILE. +-- ONLY EQL VERSION: 3 -- REQUIRE: src/v3/schema.sql -- REQUIRE: src/v3/json/types.sql -- REQUIRE: src/v3/scalars/real/query_real_types.sql diff --git a/packages/eql/src/v3/scalars/smallint/json_entry_smallint_functions.sql b/packages/eql/src/v3/scalars/smallint/json_entry_smallint_functions.sql index 44fd8e299..7f3614d94 100644 --- a/packages/eql/src/v3/scalars/smallint/json_entry_smallint_functions.sql +++ b/packages/eql/src/v3/scalars/smallint/json_entry_smallint_functions.sql @@ -1,4 +1,5 @@ -- AUTOMATICALLY GENERATED FILE. +-- ONLY EQL VERSION: 3 -- REQUIRE: src/v3/schema.sql -- REQUIRE: src/v3/json/types.sql -- REQUIRE: src/v3/json/functions.sql diff --git a/packages/eql/src/v3/scalars/smallint/json_entry_smallint_operators.sql b/packages/eql/src/v3/scalars/smallint/json_entry_smallint_operators.sql index 81ba7ff66..6adc9b996 100644 --- a/packages/eql/src/v3/scalars/smallint/json_entry_smallint_operators.sql +++ b/packages/eql/src/v3/scalars/smallint/json_entry_smallint_operators.sql @@ -1,4 +1,5 @@ -- AUTOMATICALLY GENERATED FILE. +-- ONLY EQL VERSION: 3 -- REQUIRE: src/v3/schema.sql -- REQUIRE: src/v3/json/types.sql -- REQUIRE: src/v3/scalars/smallint/query_smallint_types.sql diff --git a/packages/eql/src/v3/scalars/text/json_entry_text_functions.sql b/packages/eql/src/v3/scalars/text/json_entry_text_functions.sql index 57c5d45bc..2eb87eb1f 100644 --- a/packages/eql/src/v3/scalars/text/json_entry_text_functions.sql +++ b/packages/eql/src/v3/scalars/text/json_entry_text_functions.sql @@ -1,4 +1,5 @@ -- AUTOMATICALLY GENERATED FILE. +-- ONLY EQL VERSION: 3 -- REQUIRE: src/v3/schema.sql -- REQUIRE: src/v3/json/types.sql -- REQUIRE: src/v3/json/functions.sql diff --git a/packages/eql/src/v3/scalars/text/json_entry_text_operators.sql b/packages/eql/src/v3/scalars/text/json_entry_text_operators.sql index 029d55e8c..c4201f675 100644 --- a/packages/eql/src/v3/scalars/text/json_entry_text_operators.sql +++ b/packages/eql/src/v3/scalars/text/json_entry_text_operators.sql @@ -1,4 +1,5 @@ -- AUTOMATICALLY GENERATED FILE. +-- ONLY EQL VERSION: 3 -- REQUIRE: src/v3/schema.sql -- REQUIRE: src/v3/json/types.sql -- REQUIRE: src/v3/scalars/text/query_text_types.sql diff --git a/packages/eql/src/v3/scalars/timestamp/json_entry_timestamp_functions.sql b/packages/eql/src/v3/scalars/timestamp/json_entry_timestamp_functions.sql index 69d61924d..77acb20c2 100644 --- a/packages/eql/src/v3/scalars/timestamp/json_entry_timestamp_functions.sql +++ b/packages/eql/src/v3/scalars/timestamp/json_entry_timestamp_functions.sql @@ -1,4 +1,5 @@ -- AUTOMATICALLY GENERATED FILE. +-- ONLY EQL VERSION: 3 -- REQUIRE: src/v3/schema.sql -- REQUIRE: src/v3/json/types.sql -- REQUIRE: src/v3/json/functions.sql diff --git a/packages/eql/src/v3/scalars/timestamp/json_entry_timestamp_operators.sql b/packages/eql/src/v3/scalars/timestamp/json_entry_timestamp_operators.sql index 99c06b37e..2c0b2343b 100644 --- a/packages/eql/src/v3/scalars/timestamp/json_entry_timestamp_operators.sql +++ b/packages/eql/src/v3/scalars/timestamp/json_entry_timestamp_operators.sql @@ -1,4 +1,5 @@ -- AUTOMATICALLY GENERATED FILE. +-- ONLY EQL VERSION: 3 -- REQUIRE: src/v3/schema.sql -- REQUIRE: src/v3/json/types.sql -- REQUIRE: src/v3/scalars/timestamp/query_timestamp_types.sql diff --git a/packages/eql/tests/encryption/tests/json_tree.rs b/packages/eql/tests/encryption/tests/json_tree.rs index fe4ea4f19..0a782c083 100644 --- a/packages/eql/tests/encryption/tests/json_tree.rs +++ b/packages/eql/tests/encryption/tests/json_tree.rs @@ -1,5 +1,5 @@ //! The JSON tree family end to end: a document sealed by Stack Encrypt's -//! JSON index into `public.eql_v4_json_tree`, queried with the v4 SQL, and +//! JSON index into `public.eql_v4_json_search`, queried with the v4 SQL, and //! opened again. The unit tests need no database; the PostgreSQL test //! installs nothing itself (`mise run test:encryption:postgres` installs the //! v4 bundle built from the tree) and is the oracle the design asks for: @@ -361,7 +361,7 @@ fn postgres_containment_over_the_encrypted_column_is_jsonb_containment() { let plan = plan(); let mut db = connect(); db.batch_execute( - "CREATE TEMP TABLE json_tree_docs (id int PRIMARY KEY, plain jsonb NOT NULL, doc public.eql_v4_json_tree NOT NULL); + "CREATE TEMP TABLE json_tree_docs (id int PRIMARY KEY, plain jsonb NOT NULL, doc public.eql_v4_json_search NOT NULL); CREATE INDEX json_tree_docs_doc ON json_tree_docs USING gin ((eql_v4.json_tree_query(doc)) jsonb_path_ops);", ) .unwrap(); @@ -371,7 +371,7 @@ fn postgres_containment_over_the_encrypted_column_is_jsonb_containment() { let stored = wait(&runtime, seal(&cipher, plain)); let stored = serde_json::to_string(&stored).unwrap(); db.execute( - "INSERT INTO json_tree_docs VALUES ($1, $2::text::jsonb, $3::text::jsonb::public.eql_v4_json_tree)", + "INSERT INTO json_tree_docs VALUES ($1, $2::text::jsonb, $3::text::jsonb::public.eql_v4_json_search)", &[&(id as i32), &plain.to_string(), &stored], ) .unwrap(); @@ -396,13 +396,13 @@ fn postgres_containment_over_the_encrypted_column_is_jsonb_containment() { ); let encrypted = ids( &mut db, - "SELECT id FROM json_tree_docs WHERE doc @> $1::text::jsonb::eql_v4.query_json_tree ORDER BY id", + "SELECT id FROM json_tree_docs WHERE doc @> $1::text::jsonb::eql_v4.query_json ORDER BY id", &operand, ); assert_eq!(encrypted, expected, "@> {needle}"); let function_form = ids( &mut db, - "SELECT id FROM json_tree_docs WHERE eql_v4.json_tree_contains(doc, $1::text::jsonb::eql_v4.query_json_tree) ORDER BY id", + "SELECT id FROM json_tree_docs WHERE eql_v4.json_tree_contains(doc, $1::text::jsonb::eql_v4.query_json) ORDER BY id", &operand, ); assert_eq!(function_form, expected, "json_tree_contains {needle}"); @@ -424,7 +424,7 @@ fn postgres_containment_over_the_encrypted_column_is_jsonb_containment() { } let encrypted = ids( &mut db, - "SELECT id FROM json_tree_docs WHERE doc <@ $1::text::jsonb::eql_v4.query_json_tree ORDER BY id", + "SELECT id FROM json_tree_docs WHERE doc <@ $1::text::jsonb::eql_v4.query_json ORDER BY id", &reverse, ); assert_eq!(encrypted, expected, "<@ {needle}"); @@ -448,7 +448,7 @@ fn postgres_containment_over_the_encrypted_column_is_jsonb_containment() { let operand = serde_json::to_string(&JsonTreeQuery::try_from(&needle).unwrap()).unwrap(); let explain: Vec = db .query( - "EXPLAIN SELECT id FROM json_tree_docs WHERE doc @> $1::text::jsonb::eql_v4.query_json_tree", + "EXPLAIN SELECT id FROM json_tree_docs WHERE doc @> $1::text::jsonb::eql_v4.query_json", &[&operand], ) .unwrap() @@ -465,8 +465,8 @@ fn postgres_containment_over_the_encrypted_column_is_jsonb_containment() { // Not STRICT, so inlinable; a NULL on either side is still NULL. let nulls = db .query_one( - "SELECT (NULL::public.eql_v4_json_tree @> $1::text::jsonb::eql_v4.query_json_tree) IS NULL - AND ((SELECT doc FROM json_tree_docs WHERE id = 0) @> NULL::eql_v4.query_json_tree) IS NULL", + "SELECT (NULL::public.eql_v4_json_search @> $1::text::jsonb::eql_v4.query_json) IS NULL + AND ((SELECT doc FROM json_tree_docs WHERE id = 0) @> NULL::eql_v4.query_json) IS NULL", &[&operand], ) .unwrap(); @@ -500,7 +500,7 @@ fn postgres_extracts_walks_and_orders_by_path() { let plan = plan(); let mut db = connect(); db.batch_execute( - "CREATE TEMP TABLE json_tree_rows (id int PRIMARY KEY, doc public.eql_v4_json_tree NOT NULL)", + "CREATE TEMP TABLE json_tree_rows (id int PRIMARY KEY, doc public.eql_v4_json_search NOT NULL)", ) .unwrap(); let rows = [ @@ -515,7 +515,7 @@ fn postgres_extracts_walks_and_orders_by_path() { let stored = wait(&runtime, seal(&cipher, plain)); let stored = serde_json::to_string(&stored).unwrap(); db.execute( - "INSERT INTO json_tree_rows VALUES ($1, $2::text::jsonb::public.eql_v4_json_tree)", + "INSERT INTO json_tree_rows VALUES ($1, $2::text::jsonb::public.eql_v4_json_search)", &[id, &stored], ) .unwrap(); @@ -682,7 +682,7 @@ fn postgres_extracts_walks_and_orders_by_path() { .query_one( "SELECT count(*) FROM json_tree_rows, eql_v4.json_tree_subtree( - jsonb_set(doc::jsonb, '{h}', to_jsonb(encode(set_byte(decode(doc::jsonb ->> 'h', 'base64'), 1, 0), 'base64')))::public.eql_v4_json_tree, + jsonb_set(doc::jsonb, '{h}', to_jsonb(encode(set_byte(decode(doc::jsonb ->> 'h', 'base64'), 1, 0), 'base64')))::public.eql_v4_json_search, $1::text[], 1) e WHERE id = 1", &[&nested.tokens()], @@ -779,7 +779,7 @@ fn postgres_extracts_walks_and_orders_by_path() { let number = selector("$.number"); let ids: Vec = db .query( - "SELECT id FROM json_tree_rows WHERE (doc -> $1::text) >= $2::text::jsonb::eql_v4.query_json_tree_leaf ORDER BY id", + "SELECT id FROM json_tree_rows WHERE (doc -> $1::text) >= $2::text::jsonb::eql_v4.query_json_leaf ORDER BY id", &[&number, &at_least_one], ) .unwrap() @@ -789,7 +789,7 @@ fn postgres_extracts_walks_and_orders_by_path() { assert_eq!(ids, [1, 2]); let ids: Vec = db .query( - "SELECT id FROM json_tree_rows WHERE (doc -> $1::text) < $2::text::jsonb::eql_v4.query_json_tree_leaf ORDER BY id", + "SELECT id FROM json_tree_rows WHERE (doc -> $1::text) < $2::text::jsonb::eql_v4.query_json_leaf ORDER BY id", &[&number, &at_least_one], ) .unwrap() @@ -819,7 +819,7 @@ fn postgres_extracts_walks_and_orders_by_path() { .collect(); assert_eq!(sorted, [3, 1, 2], "aaa, world-1, world-2"); - // An ordering operand that lost its `query_json_tree_leaf` cast is plain + // An ordering operand that lost its `query_json_leaf` cast is plain // jsonb, and every comparison of an entry with plain jsonb raises, // either way round, instead of comparing the entry's ciphertext. for op in ["<", "<=", ">", ">=", "=", "<>"] { @@ -836,7 +836,7 @@ fn postgres_extracts_walks_and_orders_by_path() { .unwrap_or_else(|| error.to_string()); assert!( message.contains(&format!( - "operator {op} is not supported for public.eql_v4_json_tree_entry" + "operator {op} is not supported for public.eql_v4_json_entry" )), "{sql}: {message}" ); @@ -859,7 +859,7 @@ fn postgres_extracts_walks_and_orders_by_path() { .map(|e| e.message().to_owned()) .unwrap_or_else(|| error.to_string()); assert!( - message.contains("operator ? is not supported for public.eql_v4_json_tree"), + message.contains("operator ? is not supported for public.eql_v4_json_search"), "native `?` must be blocked: {message}" ); } @@ -989,7 +989,7 @@ fn postgres_the_planner_picks_the_indexes_without_help() { for shape in &shapes { let table = format!("json_tree_plan_{}", shape.name); db.batch_execute(&format!( - "CREATE TEMP TABLE {table} (id int PRIMARY KEY, doc public.eql_v4_json_tree NOT NULL); + "CREATE TEMP TABLE {table} (id int PRIMARY KEY, doc public.eql_v4_json_search NOT NULL); CREATE INDEX {table}_gin ON {table} USING gin ((eql_v4.json_tree_query(doc)) jsonb_path_ops); CREATE INDEX {table}_ord ON {table} (eql_v4.ord_term(doc -> '{number}'::text));" )) @@ -999,7 +999,7 @@ fn postgres_the_planner_picks_the_indexes_without_help() { let stored = serde_json::to_string(&stored).unwrap(); db.execute( &format!( - "INSERT INTO {table} VALUES ($1, $2::text::jsonb::public.eql_v4_json_tree)" + "INSERT INTO {table} VALUES ($1, $2::text::jsonb::public.eql_v4_json_search)" ), &[&(i as i32), &stored], ) @@ -1056,7 +1056,7 @@ fn postgres_the_planner_picks_the_indexes_without_help() { let operand = needle(shape.needle.clone()); let text = explain( &mut db, - &format!("SELECT id FROM {table} WHERE doc @> $1::text::jsonb::eql_v4.query_json_tree"), + &format!("SELECT id FROM {table} WHERE doc @> $1::text::jsonb::eql_v4.query_json"), &[&operand], ); assert!( @@ -1066,7 +1066,9 @@ fn postgres_the_planner_picks_the_indexes_without_help() { ); let matched: i64 = db .query_one( - &format!("SELECT count(*) FROM {table} WHERE doc @> $1::text::jsonb::eql_v4.query_json_tree"), + &format!( + "SELECT count(*) FROM {table} WHERE doc @> $1::text::jsonb::eql_v4.query_json" + ), &[&operand], ) .unwrap() @@ -1081,7 +1083,7 @@ fn postgres_the_planner_picks_the_indexes_without_help() { let operand = leaf(shape.rows - 5); let text = explain( &mut db, - &format!("SELECT id FROM {table} WHERE (doc -> $1::text) >= $2::text::jsonb::eql_v4.query_json_tree_leaf"), + &format!("SELECT id FROM {table} WHERE (doc -> $1::text) >= $2::text::jsonb::eql_v4.query_json_leaf"), &[&number, &operand], ); assert!( @@ -1092,7 +1094,7 @@ fn postgres_the_planner_picks_the_indexes_without_help() { ); let matched: i64 = db .query_one( - &format!("SELECT count(*) FROM {table} WHERE (doc -> $1::text) >= $2::text::jsonb::eql_v4.query_json_tree_leaf"), + &format!("SELECT count(*) FROM {table} WHERE (doc -> $1::text) >= $2::text::jsonb::eql_v4.query_json_leaf"), &[&number, &operand], ) .unwrap() @@ -1116,9 +1118,9 @@ fn postgres_the_planner_picks_the_indexes_without_help() { } db.batch_execute(&format!( "PREPARE {table}_contains(text) AS - SELECT id FROM {table} WHERE doc @> $1::jsonb::eql_v4.query_json_tree; + SELECT id FROM {table} WHERE doc @> $1::jsonb::eql_v4.query_json; PREPARE {table}_range(text) AS - SELECT id FROM {table} WHERE (doc -> '{number}'::text) >= $1::jsonb::eql_v4.query_json_tree_leaf;" + SELECT id FROM {table} WHERE (doc -> '{number}'::text) >= $1::jsonb::eql_v4.query_json_leaf;" )) .unwrap(); for (statement, operand, index, what) in [ @@ -1170,12 +1172,12 @@ fn postgres_the_planner_picks_the_indexes_without_help() { // The control: with `json_tree_query` priced as if it were free, the // planner takes the sequential scan over the TOASTed table. let mut tx = db.transaction().unwrap(); - tx.batch_execute("ALTER FUNCTION eql_v4.json_tree_query(public.eql_v4_json_tree) COST 1") + tx.batch_execute("ALTER FUNCTION eql_v4.json_tree_query(public.eql_v4_json_search) COST 1") .unwrap(); let operand = needle(shapes[1].needle.clone()); let text = tx .query( - "EXPLAIN SELECT id FROM json_tree_plan_nested WHERE doc @> $1::text::jsonb::eql_v4.query_json_tree", + "EXPLAIN SELECT id FROM json_tree_plan_nested WHERE doc @> $1::text::jsonb::eql_v4.query_json", &[&operand], ) .unwrap() @@ -1254,7 +1256,7 @@ fn store( documents: &[Value], ) { db.batch_execute(&format!( - "CREATE TEMP TABLE {table} (id int PRIMARY KEY, plain jsonb NOT NULL, doc public.eql_v4_json_tree NOT NULL)" + "CREATE TEMP TABLE {table} (id int PRIMARY KEY, plain jsonb NOT NULL, doc public.eql_v4_json_search NOT NULL)" )) .unwrap(); for (id, plain) in (1..).zip(documents) { @@ -1268,7 +1270,7 @@ fn store( ) .unwrap(); db.execute( - &format!("INSERT INTO {table} VALUES ($1, $2::text::jsonb, $3::text::jsonb::public.eql_v4_json_tree)"), + &format!("INSERT INTO {table} VALUES ($1, $2::text::jsonb, $3::text::jsonb::public.eql_v4_json_search)"), &[&id, &plain.to_string(), &serde_json::to_string(&stored).unwrap()], ) .unwrap(); @@ -1995,7 +1997,7 @@ fn postgres_hostile_answers_are_refused_by_the_client() { '{sv,9,x}', to_jsonb(eql_v4_internal.json_tree_self_tag( eql_v4_internal.json_tree_iv(doc::jsonb ->> 'h'), - eql_v4_internal.json_tree_member_token(decode($1::text, 'base64'), decode(doc::jsonb -> 'sv' -> 9 ->> 's', 'base64')))))::public.eql_v4_json_tree + eql_v4_internal.json_tree_member_token(decode($1::text, 'base64'), decode(doc::jsonb -> 'sv' -> 9 ->> 's', 'base64')))))::public.eql_v4_json_search WHERE id = 1", &[&nested.tokens()[0]], ) @@ -2010,7 +2012,7 @@ fn postgres_hostile_answers_are_refused_by_the_client() { let number = subtree_key("$.number"); db.execute( "UPDATE json_tree_hostile SET doc = jsonb_set(jsonb_set(doc::jsonb, '{sv,1,x}', doc::jsonb -> 'sv' -> 9 -> 'x'), - '{sv,9,x}', '\"AAAAAAAAAAA=\"')::public.eql_v4_json_tree WHERE id = 2", + '{sv,9,x}', '\"AAAAAAAAAAA=\"')::public.eql_v4_json_search WHERE id = 2", &[], ) .unwrap(); @@ -2023,7 +2025,7 @@ fn postgres_hostile_answers_are_refused_by_the_client() { // entry is found where it now sits, and fails to open there. db.execute( "UPDATE json_tree_hostile SET doc = jsonb_set(jsonb_set(doc::jsonb, - '{sv,3}', doc::jsonb -> 'sv' -> 9), '{sv,9}', doc::jsonb -> 'sv' -> 3)::public.eql_v4_json_tree + '{sv,3}', doc::jsonb -> 'sv' -> 9), '{sv,9}', doc::jsonb -> 'sv' -> 3)::public.eql_v4_json_search WHERE id = 2", &[], ) @@ -2103,7 +2105,7 @@ fn postgres_reads_never_answer_wrongly_under_tampering() { let keyset = cipher.default_keyset(); let mut db = connect(); db.batch_execute( - "CREATE TEMP TABLE json_tree_tamper (id int PRIMARY KEY, plain jsonb NOT NULL, doc public.eql_v4_json_tree NOT NULL)", + "CREATE TEMP TABLE json_tree_tamper (id int PRIMARY KEY, plain jsonb NOT NULL, doc public.eql_v4_json_search NOT NULL)", ) .unwrap(); let mut rng = Lcg(0x5eed_7a3e); @@ -2176,7 +2178,7 @@ fn postgres_reads_never_answer_wrongly_under_tampering() { _ => "a node moved under another parent", }; let inserted = db.execute( - "INSERT INTO json_tree_tamper VALUES ($1, $2::text::jsonb, $3::text::jsonb::public.eql_v4_json_tree)", + "INSERT INTO json_tree_tamper VALUES ($1, $2::text::jsonb, $3::text::jsonb::public.eql_v4_json_search)", &[&id, &plain.to_string(), &tampered.to_string()], ); if inserted.is_err() { @@ -2229,7 +2231,7 @@ fn postgres_reads_never_answer_wrongly_under_tampering() { ->> eql_v4_internal.json_tree_self_tag(eql_v4_internal.json_tree_iv(doc::jsonb ->> 'h'), decode($1::text, 'base64'))) AS j, decode(doc::jsonb -> 'sv' -> ((SELECT nodes FROM eql_v4_internal.json_tree_index(doc::jsonb)) ->> eql_v4_internal.json_tree_self_tag(eql_v4_internal.json_tree_iv(doc::jsonb ->> 'h'), decode($1::text, 'base64')))::integer ->> 's', 'base64') AS s) AS q - )::public.eql_v4_json_tree WHERE id = $5", + )::public.eql_v4_json_search WHERE id = $5", &[&from_key.tokens()[0], &to_key.tokens()[0], &width, &member, &id], ) .unwrap(); @@ -2307,7 +2309,7 @@ fn forged_chain(flags: i32, levels: i64, copies: i64, spread: bool) -> String { 'x', eql_v4_internal.json_tree_self_tag(k.iv, eql_v4_internal.json_tree_member_token(link.t, decode(repeat('00', 16), 'hex'))), 'p', eql_v4_internal.json_tree_parent_tag(k.iv, link.t, {ordinal})) ORDER BY n, copy.c) FROM link, k, copy), - 'q', '{{\"sv\": [], \"el\": []}}'::jsonb))::public.eql_v4_json_tree", + 'q', '{{\"sv\": [], \"el\": []}}'::jsonb))::public.eql_v4_json_search", last = levels - 1 ) } diff --git a/packages/eql/tests/encryption/tests/json_tree/generated.rs b/packages/eql/tests/encryption/tests/json_tree/generated.rs index 0fb88343b..006e5be22 100644 --- a/packages/eql/tests/encryption/tests/json_tree/generated.rs +++ b/packages/eql/tests/encryption/tests/json_tree/generated.rs @@ -500,7 +500,7 @@ fn scenario() -> Scenario { // sequential scans off the planner has no other index to walk whole // and filter. db.batch_execute( - "CREATE TEMP TABLE json_tree_generated (id int NOT NULL, plain jsonb NOT NULL, doc public.eql_v4_json_tree NOT NULL); + "CREATE TEMP TABLE json_tree_generated (id int NOT NULL, plain jsonb NOT NULL, doc public.eql_v4_json_search NOT NULL); CREATE INDEX json_tree_generated_doc ON json_tree_generated USING gin ((eql_v4.json_tree_query(doc)) jsonb_path_ops);", ) .unwrap(); @@ -529,7 +529,7 @@ impl Scenario { let _ = db .execute( "INSERT INTO json_tree_generated - SELECT id, plain::jsonb, doc::jsonb::public.eql_v4_json_tree + SELECT id, plain::jsonb, doc::jsonb::public.eql_v4_json_search FROM unnest($1::int[], $2::text[], $3::text[]) AS u(id, plain, doc)", &[&ids, &plains, &sealed], ) @@ -577,10 +577,7 @@ impl Scenario { scan: Scan, operands: &[String], ) -> (String, Vec>) { - let predicate = format!( - "g.doc {} n.t::jsonb::eql_v4.query_json_tree", - containment.sql() - ); + let predicate = format!("g.doc {} n.t::jsonb::eql_v4.query_json", containment.sql()); self.db.borrow_mut().batch_execute(scan.settings()).unwrap(); let explain = self .db diff --git a/packages/eql/tests/encryption/tests/json_tree_data_plan.rs b/packages/eql/tests/encryption/tests/json_tree_data_plan.rs index c9f2e5d96..a4e4326a9 100644 --- a/packages/eql/tests/encryption/tests/json_tree_data_plan.rs +++ b/packages/eql/tests/encryption/tests/json_tree_data_plan.rs @@ -5,7 +5,7 @@ //! A data-plan field names the target `EncryptedJson`, declares the `json` //! type, and hands its document over as JSON text. The engine parses it by //! one set of rules, lays it out with the JSON index under the field's -//! options, and stores the `public.eql_v4_json_tree` value; opening gives +//! options, and stores the `public.eql_v4_json_search` value; opening gives //! the document's text back. Here: //! //! - a batch of documents seals in one key request and opens to the same diff --git a/packages/eql/tests/encryption/tests/targets.rs b/packages/eql/tests/encryption/tests/targets.rs index c6e20053b..d8b5030df 100644 --- a/packages/eql/tests/encryption/tests/targets.rs +++ b/packages/eql/tests/encryption/tests/targets.rs @@ -8,7 +8,7 @@ mod common; use eql_bindings::encryption::targets::{self as targets, Opener, Target, TargetError}; use eql_bindings::v4::text::{TextEq, TextEqQuery}; use eql_bindings::{v4, Identifier}; -use eql_domains::{Shape, Term, CATALOG}; +use eql_domains::{Term, CATALOG}; use stack_encrypt::dynamic::{ValueFrame, ValueKind}; use stack_encrypt::{Decrypt, Encrypt, Label}; @@ -453,7 +453,9 @@ mod the_table { use super::*; /// The stored domains of the catalog, in catalog order: every scalar - /// domain plus the SteVec document — the rows the table must have. + /// domain — the rows the table must have before the document families'. + /// The SteVec document is EQL v3 only: in v4 its domain name, + /// `json_search`, is the JSON tree family's stored type. fn stored_domains() -> Vec<( &'static eql_domains::DomainFamily, &'static eql_domains::Domain, @@ -461,7 +463,7 @@ mod the_table { CATALOG .iter() .flat_map(|f| f.domains.iter().map(move |d| (f, d))) - .filter(|(f, d)| d.is_scalar() || d.full_name(f.name) == "json_search") + .filter(|(_, d)| d.is_scalar()) .collect() } @@ -511,14 +513,10 @@ mod the_table { format!("public.eql_v4_{}", domain.full_name(family.name)), "{name}: stored domain" ); - let indexes: Vec<&str> = if matches!(domain.shape, Shape::SteVec) { - vec!["json"] - } else { - Term::payload_terms(domain.terms) - .into_iter() - .map(index_key) - .collect() - }; + let indexes: Vec<&str> = Term::payload_terms(domain.terms) + .into_iter() + .map(index_key) + .collect(); assert_eq!(row.indexes, indexes.as_slice(), "{name}: indexes"); let suffix: String = domain .name @@ -554,14 +552,7 @@ mod the_table { kind, "{name}: the plaintext name is a vitaminc kind" ); - if matches!(domain.shape, Shape::SteVec) { - assert_eq!( - row.query, - Some("SteVecQuery"), - "{name}: the containment needle" - ); - assert_eq!(row.query_sql_domain, Some("eql_v4.query_json"), "{name}"); - } else if domain.terms.is_empty() { + if domain.terms.is_empty() { assert_eq!( (row.query, row.query_sql_domain), (None, None), @@ -586,19 +577,7 @@ mod the_table { fn every_row_is_a_compiled_domain_type_and_so_is_its_query() { let stored: Vec<&str> = v4::all().iter().map(|d| d.sql_domain()).collect(); let queries: Vec<&str> = v4::all_query().iter().map(|d| d.sql_domain()).collect(); - // v4 has no SteVec bindings yet, so the SteVec rows (the document - // and its `SteVecQuery` needle) have no compiled type behind them. - // They must stay unproducible, so neither Rust's `encrypt`/`query` - // nor a generator such as Go's can select them. for row in targets::targets() { - if row.query == Some("SteVecQuery") { - assert!( - !row.producible, - "{}: a SteVec row has no v4 type and must not be producible", - row.name - ); - continue; - } assert!( stored.contains(&row.sql_domain), "{}: {} is in all()", @@ -684,7 +663,6 @@ mod the_table { for row in targets::targets().iter().filter(|t| !t.producible) { let reason = row.reason.expect("an unproducible type has a reason"); let expected = match (row.family, row.suffix) { - ("json", "Search") => "JSON index", ("json", "") => "tree of leaves", (family, suffix) => panic!("{}: unexpected {family} {suffix}", row.name), }; diff --git a/packages/eql/tests/sqlx/tests/v3_v4_coexistence_tests.rs b/packages/eql/tests/sqlx/tests/v3_v4_coexistence_tests.rs index 7abf4480b..d92d9a1e5 100644 --- a/packages/eql/tests/sqlx/tests/v3_v4_coexistence_tests.rs +++ b/packages/eql/tests/sqlx/tests/v3_v4_coexistence_tests.rs @@ -156,11 +156,11 @@ async fn v3_and_v4_install_side_by_side(pool: PgPool) -> Result<()> { ["eql_v3", "eql_v3_internal", "eql_v4", "eql_v4_internal"] ); - // Every public domain v3 owns, v4 owns under its own prefix. v4 also - // owns the domains of the files confined to it with `-- ONLY EQL - // VERSION: 4` (`tasks/build.sh`): today the JSON tree family, which Stack - // Encrypt writes and cipherstash-client does not. Those are the only - // domains v4 may have that v3 lacks, and v3 may own nothing v4 lacks. + // Every public domain v3 owns, v4 owns under its own prefix, and v4 owns + // no other. The searchable JSON document is the one pair whose SQL + // differs: v3's `json_search` and `json_entry` are the SteVec domains + // (`src/v3/json/`, `-- ONLY EQL VERSION: 3`), v4's the JSON tree family + // (`src/v3/json_tree/`, `-- ONLY EQL VERSION: 4`) under the same names. let (v3, v4) = public_domains(&pool).await?; assert!(!v3.is_empty(), "v3 installed no public domains"); let to_v3 = |name: &String| name.replacen("eql_v4_", "eql_v3_", 1); @@ -178,10 +178,9 @@ async fn v3_and_v4_install_side_by_side(pool: PgPool) -> Result<()> { .filter(|name| !v3.contains(&to_v3(name))) .map(String::as_str) .collect(); - assert_eq!( - v4_only, - ["eql_v4_json_tree", "eql_v4_json_tree_entry"], - "the only public domains v4 owns and v3 does not are the JSON tree family" + assert!( + v4_only.is_empty(), + "v4 owns public domains v3 does not: {v4_only:?}" ); let (a, b): (String, String) = sqlx::query_as("SELECT eql_v3.version(), eql_v4.version()") diff --git a/packages/stack-encrypt/src/dynamic/record.rs b/packages/stack-encrypt/src/dynamic/record.rs index 83da9741b..2774296f5 100644 --- a/packages/stack-encrypt/src/dynamic/record.rs +++ b/packages/stack-encrypt/src/dynamic/record.rs @@ -6387,10 +6387,10 @@ mod tests { "json_tree", "", Some(ValueKind::Json), - "public.eql_v4_json_tree", + "public.eql_v4_json_search", vec!["json".to_string()], Some("EncryptedJsonQuery".to_string()), - Some("eql_v4.query_json_tree".to_string()), + Some("eql_v4.query_json".to_string()), true, None, ), diff --git a/packages/stack-encrypt/src/json/query.rs b/packages/stack-encrypt/src/json/query.rs index c7d9d4075..ea24d8ac3 100644 --- a/packages/stack-encrypt/src/json/query.rs +++ b/packages/stack-encrypt/src/json/query.rs @@ -59,7 +59,7 @@ impl JsonProjection { /// The query term of [`Contains`], [`ContainedBy`] and `AtPath`: the /// element-nested projection of the needle's selectors, and, for a needle /// whose root is a scalar, the alternative projection that matches a -/// stored root array holding it. Serializes to the `query_json_tree` +/// stored root array holding it. Serializes to the `query_json` /// operand: `{"sv": [{"s": "…", "vs": "…"}, …], "el": [{"sv": …, "el": …}, /// …], "alt": {…}}`. #[derive(Clone, Debug, PartialEq, Eq)] @@ -236,7 +236,7 @@ impl<'de> Deserialize<'de> for JsonQuery { /// The query term of `AtPath`: the selector of the /// path and the leaf ordering term of the value, which the SQL compares /// against the stored leaf's: `col -> $1 >= $2`. Serializes to the -/// `query_json_tree_leaf` operand, `{"s": "…", "op": "…"}`. `Between` +/// `query_json_leaf` operand, `{"s": "…", "op": "…"}`. `Between` /// sends two as one [`RangeQuery`]. #[derive(Clone, Debug, PartialEq, Eq)] pub struct JsonLeafQuery { From 5306f14f2087d39c6d480bd77649bb9999abb9bd Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Sun, 25 Oct 2026 23:36:55 +0000 Subject: [PATCH 10/21] feat(eql,stack-encrypt): name the v4 JSON document type JsonSearch, as v3 does The previous commit gave the JSON tree domains v3's names; this gives the contract names that point at them the same treatment. A data plan, a Go `encrypt_into` tag and the target table named the stored document `EncryptedJson`, a role name chosen before the domain had one. It is now `JsonSearch`, over `public.eql_v4_json_search`, with `JsonSearchQuery` (`eql_v4.query_json`), `JsonSearchLeafQuery` (`eql_v4.query_json_leaf`) and `JsonEntry` (`public.eql_v4_json_entry`). `Json` stays the storage-only type over `public.eql_v4_json`, so the two v3 words mean the same thing in v4. The catalog's document family (`eql_domains::JSON_TREE`) is the only place the names are written; the target table, the stack-encrypt descriptors the tests pin, the guest's producible list and the generated Go package (`eql.JSONSearch`, `JSONSearchQuery`, `JSONSearchLeafQuery`, `JSONEntry`) follow from it, regenerated with `mise run types:generate`. The Rust structs keep the implementation's name (`JsonTreeDocument` and its companions), as v3's `SteVecDocument` does. Refs #1348, #1349 Claude-Session: https://claude.ai/code/session_01HLuZxys1NDRmAkv3uhnuoJ --- languages/golang/cmd/stashgen/main_test.go | 6 +- languages/golang/encrypt/eql/eql_gen.go | 66 +++++++++---------- languages/golang/encrypt/eql/eql_test.go | 2 +- languages/golang/encrypt/guest/src/targets.rs | 2 +- .../golang/encrypt/guest/tests/native_ops.rs | 2 +- languages/golang/encrypt/json_tree_test.go | 6 +- languages/golang/internal/record/record.go | 2 +- .../golang/internal/record/record_test.go | 12 ++-- languages/golang/stashgen/read.go | 4 +- .../eql-bindings/src/encryption/targets.rs | 8 +-- .../eql/crates/eql-bindings/src/v4/targets.rs | 4 +- packages/eql/crates/eql-codegen/src/go_eql.rs | 19 +++--- packages/eql/crates/eql-domains/src/lib.rs | 22 ++++--- packages/eql/crates/eql-domains/src/tests.rs | 23 +++++-- .../encryption/tests/json_tree_data_plan.rs | 12 ++-- .../eql/tests/encryption/tests/targets.rs | 2 +- packages/stack-encrypt/src/dynamic/kind.rs | 2 +- packages/stack-encrypt/src/dynamic/mod.rs | 4 +- packages/stack-encrypt/src/dynamic/record.rs | 20 +++--- packages/stack-encrypt/src/dynamic/target.rs | 2 +- 20 files changed, 120 insertions(+), 100 deletions(-) diff --git a/languages/golang/cmd/stashgen/main_test.go b/languages/golang/cmd/stashgen/main_test.go index 42fbf9687..4e4d51297 100644 --- a/languages/golang/cmd/stashgen/main_test.go +++ b/languages/golang/cmd/stashgen/main_test.go @@ -216,13 +216,13 @@ func TestRunAsksTheEmbeddedEngine(t *testing.T) { } // A document type is refused by what is missing, not by a kind // mismatch no Go type could fix. - document := strings.Replace(userSource, "Email string `stash:\"email,encrypt_into=TextEq\"`", "Profile []byte `stash:\"profile,encrypt_into=EncryptedJson\"`", 1) + document := strings.Replace(userSource, "Email string `stash:\"email,encrypt_into=TextEq\"`", "Profile []byte `stash:\"profile,encrypt_into=JsonSearch\"`", 1) dir = writeModule(t, map[string]string{"model.go": document}) stderr.Reset() if code := run([]string{"-type", "User"}, dir, &stdout, &stderr, stashgen.GuestEngine); code != 1 { t.Fatalf("exit %d, want 1\n%s", code, stderr.String()) } - if !strings.Contains(stderr.String(), "User.Profile") || !strings.Contains(stderr.String(), "EncryptedJson seals a json document, and stashgen does not generate a field of it yet") { + if !strings.Contains(stderr.String(), "User.Profile") || !strings.Contains(stderr.String(), "JsonSearch seals a json document, and stashgen does not generate a field of it yet") { t.Fatalf("stderr = %q", stderr.String()) } // A context of two segments leaves an EQL field no column: refused by @@ -271,7 +271,7 @@ func TestEQLGoNameAgreesWithTheGeneratedPackage(t *testing.T) { t.Fatal("the rule renamed nothing: the Json family is spelled JSON") } if stashgen.EQLGoName("Json") != "JSON" || stashgen.EQLGoName("TextEq") != "TextEq" || stashgen.EQLGoName("SteVecQuery") != "SteVecQuery" || - stashgen.EQLGoName("EncryptedJsonLeafQuery") != "EncryptedJSONLeafQuery" || stashgen.EQLGoName("Jsonify") != "Jsonify" || + stashgen.EQLGoName("JsonSearchLeafQuery") != "JSONSearchLeafQuery" || stashgen.EQLGoName("Jsonify") != "Jsonify" || // A letter after the word is read as a character, as Rust reads it, // not as its first byte. stashgen.EQLGoName("Jsonécrit") != "Jsonécrit" || stashgen.EQLGoName("JsonÉcrit") != "JSONÉcrit" { diff --git a/languages/golang/encrypt/eql/eql_gen.go b/languages/golang/encrypt/eql/eql_gen.go index 50d9fa6c4..0a1111301 100644 --- a/languages/golang/encrypt/eql/eql_gen.go +++ b/languages/golang/encrypt/eql/eql_gen.go @@ -59,7 +59,7 @@ var Types = []Type{ {Name: "DoubleOrd", GoName: "DoubleOrd", Family: "double", Suffix: "Ord", Plaintext: "float64", SQLDomain: "public.eql_v4_double_ord", Indexes: []string{"ope"}, Query: "DoubleOrdQuery", QuerySQLDomain: "eql_v4.query_double_ord", Producible: true, Reason: ""}, {Name: "DoubleOrdOpe", GoName: "DoubleOrdOpe", Family: "double", Suffix: "OrdOpe", Plaintext: "float64", SQLDomain: "public.eql_v4_double_ord_ope", Indexes: []string{"ope"}, Query: "DoubleOrdOpeQuery", QuerySQLDomain: "eql_v4.query_double_ord_ope", Producible: true, Reason: ""}, {Name: "Json", GoName: "JSON", Family: "json", Suffix: "", Plaintext: "", SQLDomain: "public.eql_v4_json", Indexes: nil, Query: "", QuerySQLDomain: "", Producible: false, Reason: "a JSON value seals as a tree of leaves and EQL's `c` holds one; how a document becomes one sealed leaf is not decided"}, - {Name: "EncryptedJson", GoName: "EncryptedJSON", Family: "json_tree", Suffix: "", Plaintext: "json", SQLDomain: "public.eql_v4_json_search", Indexes: []string{"json"}, Query: "EncryptedJsonQuery", QuerySQLDomain: "eql_v4.query_json", Producible: true, Reason: ""}, + {Name: "JsonSearch", GoName: "JSONSearch", Family: "json_tree", Suffix: "", Plaintext: "json", SQLDomain: "public.eql_v4_json_search", Indexes: []string{"json"}, Query: "JsonSearchQuery", QuerySQLDomain: "eql_v4.query_json", Producible: true, Reason: ""}, } // Integer is the EQL type stored in public.eql_v4_integer: the integer family with no index. @@ -2159,96 +2159,96 @@ func (v *JSON) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// EncryptedJSON is the EQL type stored in public.eql_v4_json_search: the json_tree family with the json index. +// JSONSearch is the EQL type stored in public.eql_v4_json_search: the json_tree family with the json index. // The engine produces it from a json. -type EncryptedJSON []byte +type JSONSearch []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. -func (v EncryptedJSON) Value() (driver.Value, error) { +func (v JSONSearch) Value() (driver.Value, error) { return value(v) } // Scan reads a public.eql_v4_json_search column. -func (v *EncryptedJSON) Scan(src any) error { - return scan((*[]byte)(v), src, "EncryptedJSON") +func (v *JSONSearch) Scan(src any) error { + return scan((*[]byte)(v), src, "JSONSearch") } // MarshalJSON is the EQL value as the JSON it is, or null when empty. -func (v EncryptedJSON) MarshalJSON() ([]byte, error) { - return marshal(v, "EncryptedJSON") +func (v JSONSearch) MarshalJSON() ([]byte, error) { + return marshal(v, "JSONSearch") } // UnmarshalJSON reads the EQL value from JSON, as it is. -func (v *EncryptedJSON) UnmarshalJSON(b []byte) error { +func (v *JSONSearch) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// EncryptedJSONQuery is the query value for EncryptedJSON: the operand eql_v4.query_json takes. -type EncryptedJSONQuery []byte +// JSONSearchQuery is the query value for JSONSearch: the operand eql_v4.query_json takes. +type JSONSearchQuery []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. -func (v EncryptedJSONQuery) Value() (driver.Value, error) { +func (v JSONSearchQuery) Value() (driver.Value, error) { return value(v) } // Scan reads a eql_v4.query_json column. -func (v *EncryptedJSONQuery) Scan(src any) error { - return scan((*[]byte)(v), src, "EncryptedJSONQuery") +func (v *JSONSearchQuery) Scan(src any) error { + return scan((*[]byte)(v), src, "JSONSearchQuery") } // MarshalJSON is the EQL value as the JSON it is, or null when empty. -func (v EncryptedJSONQuery) MarshalJSON() ([]byte, error) { - return marshal(v, "EncryptedJSONQuery") +func (v JSONSearchQuery) MarshalJSON() ([]byte, error) { + return marshal(v, "JSONSearchQuery") } // UnmarshalJSON reads the EQL value from JSON, as it is. -func (v *EncryptedJSONQuery) UnmarshalJSON(b []byte) error { +func (v *JSONSearchQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// EncryptedJSONLeafQuery is the json_tree family's ordering operand at a path, compared with an entry `->` extracts: the value eql_v4.query_json_leaf holds. -type EncryptedJSONLeafQuery []byte +// JSONSearchLeafQuery is the json_tree family's ordering operand at a path, compared with an entry `->` extracts: the value eql_v4.query_json_leaf holds. +type JSONSearchLeafQuery []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. -func (v EncryptedJSONLeafQuery) Value() (driver.Value, error) { +func (v JSONSearchLeafQuery) Value() (driver.Value, error) { return value(v) } // Scan reads a eql_v4.query_json_leaf column. -func (v *EncryptedJSONLeafQuery) Scan(src any) error { - return scan((*[]byte)(v), src, "EncryptedJSONLeafQuery") +func (v *JSONSearchLeafQuery) Scan(src any) error { + return scan((*[]byte)(v), src, "JSONSearchLeafQuery") } // MarshalJSON is the EQL value as the JSON it is, or null when empty. -func (v EncryptedJSONLeafQuery) MarshalJSON() ([]byte, error) { - return marshal(v, "EncryptedJSONLeafQuery") +func (v JSONSearchLeafQuery) MarshalJSON() ([]byte, error) { + return marshal(v, "JSONSearchLeafQuery") } // UnmarshalJSON reads the EQL value from JSON, as it is. -func (v *EncryptedJSONLeafQuery) UnmarshalJSON(b []byte) error { +func (v *JSONSearchLeafQuery) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } -// EncryptedJSONEntry is the json_tree family's entry: one node of a document, as `->` and the walks return it: the value public.eql_v4_json_entry holds. -type EncryptedJSONEntry []byte +// JSONEntry is the json_tree family's entry: one node of a document, as `->` and the walks return it: the value public.eql_v4_json_entry holds. +type JSONEntry []byte // Value is the EQL value as the driver takes it: the JSON text, or NULL when empty. -func (v EncryptedJSONEntry) Value() (driver.Value, error) { +func (v JSONEntry) Value() (driver.Value, error) { return value(v) } // Scan reads a public.eql_v4_json_entry column. -func (v *EncryptedJSONEntry) Scan(src any) error { - return scan((*[]byte)(v), src, "EncryptedJSONEntry") +func (v *JSONEntry) Scan(src any) error { + return scan((*[]byte)(v), src, "JSONEntry") } // MarshalJSON is the EQL value as the JSON it is, or null when empty. -func (v EncryptedJSONEntry) MarshalJSON() ([]byte, error) { - return marshal(v, "EncryptedJSONEntry") +func (v JSONEntry) MarshalJSON() ([]byte, error) { + return marshal(v, "JSONEntry") } // UnmarshalJSON reads the EQL value from JSON, as it is. -func (v *EncryptedJSONEntry) UnmarshalJSON(b []byte) error { +func (v *JSONEntry) UnmarshalJSON(b []byte) error { return unmarshal((*[]byte)(v), b) } diff --git a/languages/golang/encrypt/eql/eql_test.go b/languages/golang/encrypt/eql/eql_test.go index 0b3df6878..815b50797 100644 --- a/languages/golang/encrypt/eql/eql_test.go +++ b/languages/golang/encrypt/eql/eql_test.go @@ -30,7 +30,7 @@ func TestTypesTableNamesTheProducibleTypes(t *testing.T) { "Boolean", "Real", "RealEq", "RealOrdOre", "RealOrd", "RealOrdOpe", "Double", "DoubleEq", "DoubleOrdOre", "DoubleOrd", "DoubleOrdOpe", - "EncryptedJson", + "JsonSearch", } if len(producible) != len(want) { t.Fatalf("producible = %v, want %v", producible, want) diff --git a/languages/golang/encrypt/guest/src/targets.rs b/languages/golang/encrypt/guest/src/targets.rs index d32ab609a..a2dfc2f04 100644 --- a/languages/golang/encrypt/guest/src/targets.rs +++ b/languages/golang/encrypt/guest/src/targets.rs @@ -249,7 +249,7 @@ mod tests { "DoubleOrdOre", "DoubleOrd", "DoubleOrdOpe", - "EncryptedJson" + "JsonSearch" ], "the engine's producible types, in catalog order" ); diff --git a/languages/golang/encrypt/guest/tests/native_ops.rs b/languages/golang/encrypt/guest/tests/native_ops.rs index e91aae234..cb2cb8e12 100644 --- a/languages/golang/encrypt/guest/tests/native_ops.rs +++ b/languages/golang/encrypt/guest/tests/native_ops.rs @@ -1413,7 +1413,7 @@ fn targets_lists_what_the_build_holds() { "DoubleOrdOre", "DoubleOrd", "DoubleOrdOpe", - "EncryptedJson" + "JsonSearch" ], "the engine's producible types, in catalog order" ); diff --git a/languages/golang/encrypt/json_tree_test.go b/languages/golang/encrypt/json_tree_test.go index 0b14d796a..2259291b1 100644 --- a/languages/golang/encrypt/json_tree_test.go +++ b/languages/golang/encrypt/json_tree_test.go @@ -14,13 +14,13 @@ import ( "github.com/cipherstash/stack/languages/golang/internal/record" ) -// An encrypted JSON document (the EQL type EncryptedJson) through the guest +// An encrypted JSON document (the EQL type JsonSearch) through the guest // that holds the EQL types: a field of kind json names the target with the // JSON index's options, crosses as its JSON text, and opens back to it. func jsonPlan() *record.Plan { return &record.Plan{Context: []string{"users"}, Fields: []record.Field{{ - Name: "profile", Kind: record.JSON, Target: "EncryptedJson", Null: record.SQLNull, + Name: "profile", Kind: record.JSON, Target: "JsonSearch", Null: record.SQLNull, TargetOptions: &record.TargetOptions{Ordered: []string{"$.age"}}, }}} } @@ -59,7 +59,7 @@ func TestJSONDocumentsRoundTripThroughTheGuest(t *testing.T) { t.Fatalf("row %d: stored %+v", i, stored) } // The value scans and binds as the generated EQL type. - var col eql.EncryptedJSON + var col eql.JSONSearch if err := col.Scan(s["profile"].EQL); err != nil { t.Fatalf("row %d: Scan: %v", i, err) } diff --git a/languages/golang/internal/record/record.go b/languages/golang/internal/record/record.go index 7f7b4b2e1..8b11cfbe6 100644 --- a/languages/golang/internal/record/record.go +++ b/languages/golang/internal/record/record.go @@ -117,7 +117,7 @@ type Field struct { Null Null // TargetOptions are the options the field gives its Target: those of // the index the target carries. A target that stores a JSON document - // (EncryptedJson) takes the JSON index's; nil runs a target under its + // (JsonSearch) takes the JSON index's; nil runs a target under its // defaults, and no other target takes any. TargetOptions *TargetOptions } diff --git a/languages/golang/internal/record/record_test.go b/languages/golang/internal/record/record_test.go index 8c7c55b19..aa0c01856 100644 --- a/languages/golang/internal/record/record_test.go +++ b/languages/golang/internal/record/record_test.go @@ -184,25 +184,25 @@ func TestParseContext(t *testing.T) { func TestWireGivesATargetItsOptions(t *testing.T) { sealed := false p := &Plan{Context: []string{"users"}, Fields: []Field{ - {Name: "profile", Kind: JSON, Target: "EncryptedJson", Null: SQLNull, TargetOptions: &TargetOptions{SealedRoot: &sealed, Ordered: []string{"$.age"}}}, - {Name: "prefs", Kind: JSON, Target: "EncryptedJson", Null: SQLNull, TargetOptions: &TargetOptions{}}, - {Name: "notes", Kind: JSON, Target: "EncryptedJson"}, + {Name: "profile", Kind: JSON, Target: "JsonSearch", Null: SQLNull, TargetOptions: &TargetOptions{SealedRoot: &sealed, Ordered: []string{"$.age"}}}, + {Name: "prefs", Kind: JSON, Target: "JsonSearch", Null: SQLNull, TargetOptions: &TargetOptions{}}, + {Name: "notes", Kind: JSON, Target: "JsonSearch"}, }} if err := p.Validate(); err != nil { t.Fatal(err) } wire := p.Wire() target := func(i int) any { return wire[i].Value.(vcvalue.Object)[1].Value } - want := vcvalue.Object{{Key: "name", Value: "EncryptedJson"}, {Key: "options", Value: vcvalue.Object{ + want := vcvalue.Object{{Key: "name", Value: "JsonSearch"}, {Key: "options", Value: vcvalue.Object{ {Key: "sealed_root", Value: false}, {Key: "ordered", Value: []any{"$.age"}}, }}} if !reflect.DeepEqual(target(0), want) { t.Fatalf("target = %#v", target(0)) } - if !reflect.DeepEqual(target(1), vcvalue.Object{{Key: "name", Value: "EncryptedJson"}, {Key: "options", Value: vcvalue.Object{}}}) { + if !reflect.DeepEqual(target(1), vcvalue.Object{{Key: "name", Value: "JsonSearch"}, {Key: "options", Value: vcvalue.Object{}}}) { t.Fatalf("empty options = %#v", target(1)) } - if target(2) != "EncryptedJson" { + if target(2) != "JsonSearch" { t.Fatalf("no options = %#v", target(2)) } if typ := wire[0].Value.(vcvalue.Object)[2]; typ.Key != "type" || typ.Value != "json" { diff --git a/languages/golang/stashgen/read.go b/languages/golang/stashgen/read.go index bc35ba64d..18e20f60c 100644 --- a/languages/golang/stashgen/read.go +++ b/languages/golang/stashgen/read.go @@ -151,8 +151,8 @@ type modelField struct { // EQLGoName is the Go type name in encrypt/eql of an EQL type the engine // names: the same name in every language, save that the word Json is JSON, -// as Go spells initialisms (Json is JSON, EncryptedJsonQuery is -// EncryptedJSONQuery; Json is a word where the name ends or the next +// as Go spells initialisms (Json is JSON, JsonSearchQuery is +// JSONSearchQuery; Json is a word where the name ends or the next // letter is upper case). The // generated encrypt/eql package follows the same rule (eql-codegen's go_eql // renderer), and its Types table records both names, which is what holds diff --git a/packages/eql/crates/eql-bindings/src/encryption/targets.rs b/packages/eql/crates/eql-bindings/src/encryption/targets.rs index c7af38a13..1e6e100b8 100644 --- a/packages/eql/crates/eql-bindings/src/encryption/targets.rs +++ b/packages/eql/crates/eql-bindings/src/encryption/targets.rs @@ -51,7 +51,7 @@ //! | `plaintext` | The vitaminc [`ValueKind`] name the type is produced from — the same names a plan field's `"type"` key uses — or `null` while the family's plaintext encoding for the stack-encrypt producer profile is unspecified. | //! | `sql_domain` | The schema-qualified PostgreSQL domain the stored value inhabits. | //! | `indexes` | The indexes the type carries, by the engine's `IndexSpec::key()` names: `eq`, `match`, `ore`, `ope`, and `json` for a document family's stored type. A query may ask a field typed with this target for exactly these. | -//! | `query` | The query twin's type name (`TextEqQuery`, `EncryptedJsonQuery`), or `null` for a storage-only type, which answers no query. | +//! | `query` | The query twin's type name (`TextEqQuery`, `JsonSearchQuery`), or `null` for a storage-only type, which answers no query. | //! | `query_sql_domain` | The query twin's PostgreSQL domain, `null` likewise. | //! | `producible` | Whether the engine can produce this type today. [`encrypt`] and [`query`] refuse a type that is not, and a generator should too. | //! | `reason` | Why not, when `producible` is `false`; `null` when it is. | @@ -349,7 +349,7 @@ pub fn encrypt<'a, K: 'static>( /// [`encrypt`], under the options the plan gives the field /// ([`TargetOptions`]): a type that stores a JSON document -/// (`EncryptedJson`) seals under the JSON index's options, the defaults +/// (`JsonSearch`) seals under the JSON index's options, the defaults /// when `options` is `None`. Every other type takes none. /// /// # Errors @@ -412,7 +412,7 @@ fn document(name: &str) -> Option<&'static Target> { target(name).filter(|target| crate::v4::targets::DOCUMENT_FAMILIES.contains(&target.family)) } -/// Seal one `EncryptedJson` value: the document the field's text holds, +/// Seal one `JsonSearch` value: the document the field's text holds, /// parsed by the engine's rules, laid out by the JSON index under /// `options`, under the column. The parsed document is wiped when this /// returns; the pending has taken what it seals. @@ -439,7 +439,7 @@ fn encrypt_document<'a, K: 'static>( .try_map(|value| serde_json::to_vec(&value).map_err(|error| Error::Other(Box::new(error))))) } -/// Open one stored `EncryptedJson` value to the document's text, checking +/// Open one stored `JsonSearch` value to the document's text, checking /// it was stored under the column. fn decrypt_document<'a, K: 'static>( name: &'static str, diff --git a/packages/eql/crates/eql-bindings/src/v4/targets.rs b/packages/eql/crates/eql-bindings/src/v4/targets.rs index abecea153..258668916 100644 --- a/packages/eql/crates/eql-bindings/src/v4/targets.rs +++ b/packages/eql/crates/eql-bindings/src/v4/targets.rs @@ -620,13 +620,13 @@ pub const TARGETS: &[Target] = &[ ), }, Target { - name: "EncryptedJson", + name: "JsonSearch", family: "json_tree", suffix: "", plaintext: Some("json"), sql_domain: "public.eql_v4_json_search", indexes: &["json"], - query: Some("EncryptedJsonQuery"), + query: Some("JsonSearchQuery"), query_sql_domain: Some("eql_v4.query_json"), producible: true, reason: None, diff --git a/packages/eql/crates/eql-codegen/src/go_eql.rs b/packages/eql/crates/eql-codegen/src/go_eql.rs index 01ddeb29b..2ecbf63d5 100644 --- a/packages/eql/crates/eql-codegen/src/go_eql.rs +++ b/packages/eql/crates/eql-codegen/src/go_eql.rs @@ -36,7 +36,7 @@ pub const GO_GENERATED_MARKER: &str = /// The Go type name of a catalog name: the same, save that the word /// `Json` is spelled `JSON`, as Go spells initialisms: `Json` is `JSON`, -/// `EncryptedJsonQuery` is `EncryptedJSONQuery`. `Json` is a word where +/// `JsonSearchQuery` is `JSONSearchQuery`. `Json` is a word where /// it ends the name or the next letter starts another (upper case). pub fn go_name(name: &str) -> String { let mut out = String::with_capacity(name.len()); @@ -252,8 +252,9 @@ mod tests { assert_eq!(go_name("TextEqQuery"), "TextEqQuery"); assert_eq!(go_name("Json"), "JSON"); assert_eq!(go_name("SteVecDocument"), "SteVecDocument"); - assert_eq!(go_name("EncryptedJson"), "EncryptedJSON"); - assert_eq!(go_name("EncryptedJsonLeafQuery"), "EncryptedJSONLeafQuery"); + assert_eq!(go_name("JsonSearch"), "JSONSearch"); + assert_eq!(go_name("JsonSearchLeafQuery"), "JSONSearchLeafQuery"); + assert_eq!(go_name("JsonEntry"), "JSONEntry"); assert_eq!(go_name("Jsonify"), "Jsonify", "a word, not a prefix"); assert_eq!( go_name("Jsonécrit"), @@ -267,10 +268,10 @@ mod tests { fn the_document_family_has_its_go_types() { let out = render_go_eql(); for go in [ - "EncryptedJSON", - "EncryptedJSONQuery", - "EncryptedJSONLeafQuery", - "EncryptedJSONEntry", + "JSONSearch", + "JSONSearchQuery", + "JSONSearchLeafQuery", + "JSONEntry", ] { assert_eq!( out.matches(&format!("\ntype {go} []byte\n")).count(), @@ -279,9 +280,9 @@ mod tests { ); } assert!(out.contains( - "Name: \"EncryptedJson\", GoName: \"EncryptedJSON\", Family: \"json_tree\", Suffix: \"\", \ + "Name: \"JsonSearch\", GoName: \"JSONSearch\", Family: \"json_tree\", Suffix: \"\", \ Plaintext: \"json\", SQLDomain: \"public.eql_v4_json_search\", Indexes: []string{\"json\"}, \ - Query: \"EncryptedJsonQuery\", QuerySQLDomain: \"eql_v4.query_json\", Producible: true" + Query: \"JsonSearchQuery\", QuerySQLDomain: \"eql_v4.query_json\", Producible: true" )); } diff --git a/packages/eql/crates/eql-domains/src/lib.rs b/packages/eql/crates/eql-domains/src/lib.rs index 8baafcb2f..0c5b95e10 100644 --- a/packages/eql/crates/eql-domains/src/lib.rs +++ b/packages/eql/crates/eql-domains/src/lib.rs @@ -577,7 +577,7 @@ pub struct DocumentDomain { pub role: DocumentRole, /// The type's name across languages: what a data plan names as a /// target, and what eql-codegen derives the Go name from - /// (`EncryptedJson` is `eql.EncryptedJSON`). A contract identifier, + /// (`JsonSearch` is `eql.JSONSearch`). A contract identifier, /// not the Rust struct's name. pub name: &'static str, /// The hand-written Rust struct in `eql_bindings::v4::`. @@ -616,9 +616,15 @@ impl DocumentFamily { /// `json_tree`: an encrypted JSON document stored as one entry per node, /// with exact containment, equality and ordering at paths, `->` extraction -/// and subtree walks (`docs/plans/2026-10-17-json-tree-index.md`). The -/// stored type is named for its role, `EncryptedJson`, over the SQL domain -/// `public.eql_v4_json_search`; the Rust struct keeps its family name. +/// and subtree walks (`docs/plans/2026-10-17-json-tree-index.md`). It is +/// EQL v4's searchable JSON document, so its types take the names EQL v3 +/// gives its own (`public.eql_v3_json_search`, the SteVec document): +/// `JsonSearch` over `public.eql_v4_json_search`, its needle +/// `JsonSearchQuery` (`eql_v4.query_json`), its ordering operand +/// `JsonSearchLeafQuery` (`eql_v4.query_json_leaf`) and its entry +/// `JsonEntry` (`public.eql_v4_json_entry`). The storage-only `Json` +/// (`public.eql_v4_json`) is the `json` family's. The Rust structs keep the +/// implementation's name, as v3's `SteVecDocument` does. pub const JSON_TREE: DocumentFamily = DocumentFamily { name: "json_tree", plaintext: "json", @@ -626,25 +632,25 @@ pub const JSON_TREE: DocumentFamily = DocumentFamily { domains: &[ DocumentDomain { role: DocumentRole::Stored, - name: "EncryptedJson", + name: "JsonSearch", rust: "JsonTreeDocument", sql_domain: "public.eql_v4_json_search", }, DocumentDomain { role: DocumentRole::Query, - name: "EncryptedJsonQuery", + name: "JsonSearchQuery", rust: "JsonTreeQuery", sql_domain: "eql_v4.query_json", }, DocumentDomain { role: DocumentRole::LeafQuery, - name: "EncryptedJsonLeafQuery", + name: "JsonSearchLeafQuery", rust: "JsonTreeLeafQuery", sql_domain: "eql_v4.query_json_leaf", }, DocumentDomain { role: DocumentRole::Entry, - name: "EncryptedJsonEntry", + name: "JsonEntry", rust: "JsonTreeEntry", sql_domain: "public.eql_v4_json_entry", }, diff --git a/packages/eql/crates/eql-domains/src/tests.rs b/packages/eql/crates/eql-domains/src/tests.rs index 00d4d42a7..a59b855c7 100644 --- a/packages/eql/crates/eql-domains/src/tests.rs +++ b/packages/eql/crates/eql-domains/src/tests.rs @@ -1663,10 +1663,25 @@ mod document_families { } #[test] - fn json_tree_is_the_encrypted_json_target() { - let stored = JSON_TREE.domain(DocumentRole::Stored).unwrap(); - assert_eq!(stored.name, "EncryptedJson"); - assert_eq!(stored.sql_domain, "public.eql_v4_json_search"); + fn json_tree_is_the_json_search_target() { + // The names EQL v3 gives its searchable JSON document, in v4. + for (role, name, sql_domain) in [ + ( + DocumentRole::Stored, + "JsonSearch", + "public.eql_v4_json_search", + ), + (DocumentRole::Query, "JsonSearchQuery", "eql_v4.query_json"), + ( + DocumentRole::LeafQuery, + "JsonSearchLeafQuery", + "eql_v4.query_json_leaf", + ), + (DocumentRole::Entry, "JsonEntry", "public.eql_v4_json_entry"), + ] { + let domain = JSON_TREE.domain(role).unwrap(); + assert_eq!((domain.name, domain.sql_domain), (name, sql_domain)); + } assert_eq!(JSON_TREE.plaintext, "json"); assert_eq!(JSON_TREE.index, "json"); } diff --git a/packages/eql/tests/encryption/tests/json_tree_data_plan.rs b/packages/eql/tests/encryption/tests/json_tree_data_plan.rs index a4e4326a9..c564093af 100644 --- a/packages/eql/tests/encryption/tests/json_tree_data_plan.rs +++ b/packages/eql/tests/encryption/tests/json_tree_data_plan.rs @@ -2,7 +2,7 @@ //! takes (the Go guest installs the same resolver), against the typed Rust //! path it must agree with. //! -//! A data-plan field names the target `EncryptedJson`, declares the `json` +//! A data-plan field names the target `JsonSearch`, declares the `json` //! type, and hands its document over as JSON text. The engine parses it by //! one set of rules, lays it out with the JSON index under the field's //! options, and stores the `public.eql_v4_json_search` value; opening gives @@ -105,7 +105,7 @@ fn options() -> JsonOptions { } /// A plan with one `json` field, `profile`, under `users/profile`, naming -/// `EncryptedJson` with [`options`], as a binding writes it. +/// `JsonSearch` with [`options`], as a binding writes it. fn plan() -> record::Plan { let wire = json!({ "profile": { @@ -113,7 +113,7 @@ fn plan() -> record::Plan { "type": "json", "null": "sql", "target": { - "name": "EncryptedJson", + "name": "JsonSearch", "options": {"ordered": ["$.age", "$.visits[*].at"]} } } @@ -344,7 +344,7 @@ async fn a_document_has_no_one_plaintext_query() { let keyset = cipher.default_keyset(); let label: Label = "users/profile".parse().unwrap(); let error = eql_targets::query( - "EncryptedJson", + "JsonSearch", &stack_encrypt::target::IndexSpec::Json(JsonOptions::default()), &keyset, &label, @@ -355,9 +355,7 @@ async fn a_document_has_no_one_plaintext_query() { assert!( matches!( error, - eql_targets::TargetError::DocumentQuery { - name: "EncryptedJson" - } + eql_targets::TargetError::DocumentQuery { name: "JsonSearch" } ), "{error}" ); diff --git a/packages/eql/tests/encryption/tests/targets.rs b/packages/eql/tests/encryption/tests/targets.rs index d8b5030df..d94259937 100644 --- a/packages/eql/tests/encryption/tests/targets.rs +++ b/packages/eql/tests/encryption/tests/targets.rs @@ -657,7 +657,7 @@ mod the_table { "DoubleOrdOre", "DoubleOrd", "DoubleOrdOpe", - "EncryptedJson" + "JsonSearch" ], ); for row in targets::targets().iter().filter(|t| !t.producible) { diff --git a/packages/stack-encrypt/src/dynamic/kind.rs b/packages/stack-encrypt/src/dynamic/kind.rs index 5381ec49e..da917dc93 100644 --- a/packages/stack-encrypt/src/dynamic/kind.rs +++ b/packages/stack-encrypt/src/dynamic/kind.rs @@ -397,7 +397,7 @@ pub enum ValueKind { /// ([`crate::json::parse_text`]) and opens back to as bytes. A value /// is never of this kind ([`ValueFrame::kind`] says `string` or /// `bytes`); a field declares it, and takes a target that stores a - /// document (`EncryptedJson`). A JSON document's `null` is the text + /// document (`JsonSearch`). A JSON document's `null` is the text /// `null`, not the frame's null. Json, } diff --git a/packages/stack-encrypt/src/dynamic/mod.rs b/packages/stack-encrypt/src/dynamic/mod.rs index 1f82a9bb5..ccfdd22a0 100644 --- a/packages/stack-encrypt/src/dynamic/mod.rs +++ b/packages/stack-encrypt/src/dynamic/mod.rs @@ -551,12 +551,12 @@ reasons! { IndexNotAdmitted => "index_not_admitted", "the declared type has no such index"; /// The field declares the JSON index as an output. A data plan /// reaches the JSON index through a target that stores a document - /// (`EncryptedJson`, with the `json` type), whose options are the + /// (`JsonSearch`, with the `json` type), whose options are the /// index's; the index is not an output of its own. JsonIndexPending => "json_index_pending", "the JSON index is reached through a target that stores a document, not as an output"; /// The field declares the `json` type and names no target: a JSON /// document is stored through a target that stores one - /// (`EncryptedJson`). + /// (`JsonSearch`). JsonNeedsTarget => "json_needs_target", r#"a "json" field names a target that stores a document"#; /// The field gives its target options the target does not take: a /// target takes the options of the index it carries, and this one diff --git a/packages/stack-encrypt/src/dynamic/record.rs b/packages/stack-encrypt/src/dynamic/record.rs index 2774296f5..dd1a60b9e 100644 --- a/packages/stack-encrypt/src/dynamic/record.rs +++ b/packages/stack-encrypt/src/dynamic/record.rs @@ -1113,11 +1113,11 @@ pub fn plan(frame: &[u8]) -> Result { /// /// `"target"` may instead be an object of the name and the options of the /// index the type carries: a type that stores a JSON document -/// (`EncryptedJson`) takes the JSON index's options object, as the +/// (`JsonSearch`) takes the JSON index's options object, as the /// `{"json": ...}` output spells it, and no other type takes any. /// /// ```text -/// "target": { "name": "EncryptedJson", +/// "target": { "name": "JsonSearch", /// "options": { "sealed_root": , "ordered": [, ...] } } /// ``` /// @@ -6383,13 +6383,13 @@ mod tests { None, ), TargetDescriptor::new( - "EncryptedJson", + "JsonSearch", "json_tree", "", Some(ValueKind::Json), "public.eql_v4_json_search", vec!["json".to_string()], - Some("EncryptedJsonQuery".to_string()), + Some("JsonSearchQuery".to_string()), Some("eql_v4.query_json".to_string()), true, None, @@ -6568,7 +6568,7 @@ mod tests { let bare = plan_with( obj(vec![( "profile", - target_spec(label("profile"), "EncryptedJson"), + target_spec(label("profile"), "JsonSearch"), )]), &FakeEql, ) @@ -6591,7 +6591,7 @@ mod tests { ("context", label("profile")), ( "target", - obj(vec![("name", s("EncryptedJson")), ("options", options)]), + obj(vec![("name", s("JsonSearch")), ("options", options)]), ), ]), )]), @@ -6650,7 +6650,7 @@ mod tests { assert_eq!(error.reason(), Some(Reason::TargetOptions), "{error}"); // A present `"options"` holds an object: a null is refused, // never read as no options. - for target in [TEXT_EQ, "EncryptedJson"] { + for target in [TEXT_EQ, "JsonSearch"] { let error = refused(plan_with( obj(vec![( "email", @@ -6699,7 +6699,7 @@ mod tests { &FakeEql, )); assert_eq!(error.reason(), Some(Reason::JsonNeedsTarget), "{error}"); - let mut spec = target_spec(label("profile"), "EncryptedJson"); + let mut spec = target_spec(label("profile"), "JsonSearch"); if let Tree::Object(entries) = &mut spec { entries.push(("null".to_string(), s("sealed"))); } @@ -6715,7 +6715,7 @@ mod tests { /// the check, before any key, with the code encrypt gives it. #[test] fn the_check_reads_a_document_by_the_engines_rules() { - let mut spec = target_spec(label("profile"), "EncryptedJson"); + let mut spec = target_spec(label("profile"), "JsonSearch"); if let Tree::Object(entries) = &mut spec { entries.push(("null".to_string(), s("sql"))); } @@ -6736,7 +6736,7 @@ mod tests { }; assert_eq!( (name.as_str(), target.as_str(), *reason), - ("profile", "EncryptedJson", code) + ("profile", "JsonSearch", code) ); assert!(!error.to_string().contains("secret"), "{error}"); } diff --git a/packages/stack-encrypt/src/dynamic/target.rs b/packages/stack-encrypt/src/dynamic/target.rs index 3fcf97abe..fe0c30a4d 100644 --- a/packages/stack-encrypt/src/dynamic/target.rs +++ b/packages/stack-encrypt/src/dynamic/target.rs @@ -212,7 +212,7 @@ impl TargetDescriptor { /// The options a plan field gives the EQL type it names: the options of /// the index the type carries. A target that stores a JSON document -/// (`EncryptedJson`) carries the JSON index and takes its options, the +/// (`JsonSearch`) carries the JSON index and takes its options, the /// sealed root and the ordered paths; no other target takes any today. /// /// The plan resolves them once, when it is built ([`Plan::new_with`]): From 2b785599f8dee1dbd1f0dd5742a8ab7c38aead72 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Sun, 25 Oct 2026 23:37:04 +0000 Subject: [PATCH 11/21] docs(eql,stack-encrypt): record the v3 names the v4 JSON document takes The changesets and CHANGELOGs for the unreleased JSON tree family named the target `EncryptedJson` over `public.eql_v4_json_tree`, and the target changeset said no SQL changed. They now give the names the family ships under, say that the v4 bundle no longer carries the SteVec domains and that the v3 bundle gains only the version-directive comment lines, and the Go CHANGELOG lists the removal of `eql.SteVecDocument` and `eql.SteVecQuery` under Breaking. AGENTS.md and packages/eql/AGENTS.md record which surfaces are confined to which version; the design plan records the decision and keeps its earlier sections' placeholder names as history. Refs #1348, #1349 Claude-Session: https://claude.ai/code/session_01HLuZxys1NDRmAkv3uhnuoJ --- .changeset/eql-encrypted-json-target.md | 2 +- .changeset/eql-json-tree-domain.md | 2 +- .../eql-json-tree-walk-and-projection.md | 2 +- AGENTS.md | 8 ++++++++ docs/plans/2026-10-17-json-tree-index.md | 4 ++-- languages/golang/CHANGELOG.md | 18 +++++++++++++----- packages/eql/AGENTS.md | 4 ++-- packages/eql/crates/eql-bindings/CHANGELOG.md | 13 +++++++++---- packages/stack-encrypt/CHANGELOG.md | 4 ++-- 9 files changed, 39 insertions(+), 18 deletions(-) diff --git a/.changeset/eql-encrypted-json-target.md b/.changeset/eql-encrypted-json-target.md index 71e8436bb..d27a5adca 100644 --- a/.changeset/eql-encrypted-json-target.md +++ b/.changeset/eql-encrypted-json-target.md @@ -2,4 +2,4 @@ "@cipherstash/eql": minor --- -**The `json_tree` family is in the EQL catalog, and a data plan can produce `public.eql_v4_json_tree` by name: the target `EncryptedJson`.** A language SDK names it in a plan field with the `json` type, gives it the JSON index's options (`{"name": "EncryptedJson", "options": {"sealed_root": true, "ordered": ["$.age"]}}`, or the bare name for the defaults), and hands each document over as UTF-8 JSON text; the `eql-bindings` crate's by-name dispatch (`encryption::targets::encrypt_with`, `decrypt`) seals it through the same JSON index layout typed Rust uses and opens it back to its text. The text is parsed by Stack Encrypt's rules, before any key is requested: an object that names a member twice, an integer literal outside the 64-bit range, and text that is not JSON are refused (`TargetError::Json`, with a code and never a byte of the text). The catalog lists the family once (`eql_domains::DOCUMENT_FAMILIES`), so the target table, `v4::all()` / `v4::all_query()` and the Go types (`eql.EncryptedJSON`, `EncryptedJSONQuery`, `EncryptedJSONLeafQuery`, `EncryptedJSONEntry`) are generated from it. No SQL changes. +**The `json_tree` family is in the EQL catalog as EQL v4's searchable JSON document, and a data plan can produce `public.eql_v4_json_search` by name: the target `JsonSearch`.** A language SDK names it in a plan field with the `json` type, gives it the JSON index's options (`{"name": "JsonSearch", "options": {"sealed_root": true, "ordered": ["$.age"]}}`, or the bare name for the defaults), and hands each document over as UTF-8 JSON text; the `eql-bindings` crate's by-name dispatch (`encryption::targets::encrypt_with`, `decrypt`) seals it through the same JSON index layout typed Rust uses and opens it back to its text. The text is parsed by Stack Encrypt's rules, before any key is requested: an object that names a member twice, an integer literal outside the 64-bit range, and text that is not JSON are refused (`TargetError::Json`, with a code and never a byte of the text). The catalog lists the family once (`eql_domains::DOCUMENT_FAMILIES`), so the target table, `v4::all()` / `v4::all_query()` and the Go types (`eql.JSONSearch`, `JSONSearchQuery`, `JSONSearchLeafQuery`, `JSONEntry`) are generated from it. The types take the names EQL v3 gives its searchable JSON document: `Json` (`public.eql_v4_json`) stays the storage-only type, `JsonSearch` is the searchable document, queried with `JsonSearchQuery` (`eql_v4.query_json`) and `JsonSearchLeafQuery` (`eql_v4.query_json_leaf`), and `JsonEntry` (`public.eql_v4_json_entry`) is one node as `->` and the reads return it. The JSON tree SQL domains are renamed to match: `public.eql_v4_json_tree` is now `public.eql_v4_json_search`, `public.eql_v4_json_tree_entry` is `public.eql_v4_json_entry`, `eql_v4.query_json_tree` is `eql_v4.query_json` and `eql_v4.query_json_tree_leaf` is `eql_v4.query_json_leaf`; the `eql_v4.json_tree_*` functions keep their names. The v4 bundle no longer carries the SteVec domains: in v4, `json_tree` implements `json_search`. The SteVec SQL (`src/v3/json/`) and the generated `json_entry_` cross operators are confined to EQL v3 with `-- ONLY EQL VERSION: 3`, and the v4 target table drops its unproducible `SteVecDocument` row. The v3 bundle gains only those directive comment lines. diff --git a/.changeset/eql-json-tree-domain.md b/.changeset/eql-json-tree-domain.md index 1616779ff..16e5a2ca1 100644 --- a/.changeset/eql-json-tree-domain.md +++ b/.changeset/eql-json-tree-domain.md @@ -2,6 +2,6 @@ "@cipherstash/eql": minor --- -**EQL v4 gains the `json_tree` domain family: `public.eql_v4_json_tree`, an encrypted JSON document stored as one entry per node, with `@>` and `<@` that agree with PostgreSQL's own `jsonb @>` across array elements.** The family is new beside `public.eql_v4_json` and is not a port of the v3 SteVec surface: it is what Stack Encrypt's JSON index writes. A document's entries each carry a path selector `s` and a value selector `vs` (containment is a set comparison over the pair), the node's own ciphertext `c` (its label, kind and scalar value; a document's ciphertext is linear in its size, where a SteVec document repeats every subtree at every level), and a token chain (`p`, `w`, `o`) that lets core PostgreSQL walk the subtree under a node from one client-derived token: `eql_v4.json_tree_subtree(doc, token)`, `json_tree_subtree_at(doc, token, selector)`, `json_tree_children` and `json_tree_child_count`. By default the root is sealed, so no query can hand the server a token for the whole column. Containment projects a stored document into an element-nested shape from its group tags (`n`, `g`), `eql_v4.json_tree_query(doc)`, and compares it with the needle (`eql_v4.query_json_tree`) by native `jsonb @>`, so each needle array element must fit inside one stored element, which closes the cross-element false positives of selector-set containment (#1231) for this family; a GIN index on `(eql_v4.json_tree_query(col)::jsonb) jsonb_path_ops` serves it. `->` by selector returns an entry that opens on its own (`public.eql_v4_json_tree_entry`, with the header and ordinal grafted), and `eql_v4.query_json_tree_leaf` orders an extracted number or string leaf with `<`, `<=`, `>`, `>=`. Every other native jsonb operator on the column is blocked, as on the other EQL domains. EQL v4 only (`-- ONLY EQL VERSION: 4`); the v3 bundle is unchanged. The `eql-bindings` crate carries the hand-written `v4::json_tree` types with Stack Encrypt's `EncryptFrom` / `DecryptInto` for the document and the entry. +**EQL v4 gains the `json_tree` domain family: `public.eql_v4_json_search`, an encrypted JSON document stored as one entry per node, with `@>` and `<@` that agree with PostgreSQL's own `jsonb @>` across array elements.** The family is new beside `public.eql_v4_json` and is not a port of the v3 SteVec surface: it is what Stack Encrypt's JSON index writes. A document's entries each carry a path selector `s` and a value selector `vs` (containment is a set comparison over the pair), the node's own ciphertext `c` (its label, kind and scalar value; a document's ciphertext is linear in its size, where a SteVec document repeats every subtree at every level), and a token chain (`p`, `w`, `o`) that lets core PostgreSQL walk the subtree under a node from one client-derived token: `eql_v4.json_tree_subtree(doc, token)`, `json_tree_subtree_at(doc, token, selector)`, `json_tree_children` and `json_tree_child_count`. By default the root is sealed, so no query can hand the server a token for the whole column. Containment projects a stored document into an element-nested shape from its group tags (`n`, `g`), `eql_v4.json_tree_query(doc)`, and compares it with the needle (`eql_v4.query_json`) by native `jsonb @>`, so each needle array element must fit inside one stored element, which closes the cross-element false positives of selector-set containment (#1231) for this family; a GIN index on `(eql_v4.json_tree_query(col)::jsonb) jsonb_path_ops` serves it. `->` by selector returns an entry that opens on its own (`public.eql_v4_json_entry`, with the header and ordinal grafted), and `eql_v4.query_json_leaf` orders an extracted number or string leaf with `<`, `<=`, `>`, `>=`. Every other native jsonb operator on the column is blocked, as on the other EQL domains. EQL v4 only (`-- ONLY EQL VERSION: 4`); the v3 bundle's SQL is unchanged. The `eql-bindings` crate carries the hand-written `v4::json_tree` types with Stack Encrypt's `EncryptFrom` / `DecryptInto` for the document and the entry. Why. Stack Encrypt had no JSON index: the operators (`Contains`, `ContainedBy`, `AtPath`) were declared and answered by nothing, and the v4 JSON wire format was unconstrained by anything shipped. This is Phase 1 and 2 of the `json_tree` index's phased plan (#1249), implemented against `docs/plans/2026-10-17-json-tree-index.md`, whose "Implementation notes" section records where the code deviates from the design. diff --git a/.changeset/eql-json-tree-walk-and-projection.md b/.changeset/eql-json-tree-walk-and-projection.md index 4e900b9cb..eb03d448f 100644 --- a/.changeset/eql-json-tree-walk-and-projection.md +++ b/.changeset/eql-json-tree-walk-and-projection.md @@ -4,7 +4,7 @@ **The `json_tree` family's read path no longer pays for the document's size per query.** Three changes to the EQL v4 `json_tree` SQL, each measured (cipherstash/benches, `report/JSON_TREE_COMPARISON.md` at 57abcf9) on a 10,000-row corpus of nested documents (about 220 nodes each) where the shipped surface was one to four orders of magnitude behind native `jsonb`: -- **A stored document carries its own containment projection, `q`.** The client writes the document's `{s, vs}` pairs nested by array element beside `sv`, and `eql_v4.json_tree_query(doc)` now reads that field instead of rebuilding the projection from the group tags on every call. The GIN index recheck, which runs the projection once per candidate row, was the whole cost of `@>` on a nested document: 75 ms for ten rows, 16 s for a two-thousand-row result set, 44 s to build the index on 10,000 rows. With the projection stored the same queries measure in the low milliseconds and the index builds in seconds. `json_tree_query` returns `jsonb` rather than the needle domain, since the domain's CHECK would run per recheck; the document's own CHECK validates `q` when the row is written. It is a plpgsql function costed at 1000 rather than an inlinable SQL one: inlined to `value -> 'q'` the planner costs the filter as one operator, and a column of large documents is TOASTed out of the heap, so a sequential scan over it looked nearly free and was chosen over the GIN index while in fact decompressing every row (1.5 s for a one-row match on 10,000 rows; 0.5 to 0.7 ms through the index). The index expression is `(eql_v4.json_tree_query(col)) jsonb_path_ops`; the `::jsonb` cast earlier guidance wrote is harmless and no longer needed. **Wire change:** `q` is required in `public.eql_v4_json_tree`, and a document written by the earlier build of this unreleased family is refused by the domain CHECK. +- **A stored document carries its own containment projection, `q`.** The client writes the document's `{s, vs}` pairs nested by array element beside `sv`, and `eql_v4.json_tree_query(doc)` now reads that field instead of rebuilding the projection from the group tags on every call. The GIN index recheck, which runs the projection once per candidate row, was the whole cost of `@>` on a nested document: 75 ms for ten rows, 16 s for a two-thousand-row result set, 44 s to build the index on 10,000 rows. With the projection stored the same queries measure in the low milliseconds and the index builds in seconds. `json_tree_query` returns `jsonb` rather than the needle domain, since the domain's CHECK would run per recheck; the document's own CHECK validates `q` when the row is written. It is a plpgsql function costed at 1000 rather than an inlinable SQL one: inlined to `value -> 'q'` the planner costs the filter as one operator, and a column of large documents is TOASTed out of the heap, so a sequential scan over it looked nearly free and was chosen over the GIN index while in fact decompressing every row (1.5 s for a one-row match on 10,000 rows; 0.5 to 0.7 ms through the index). The index expression is `(eql_v4.json_tree_query(col)) jsonb_path_ops`; the `::jsonb` cast earlier guidance wrote is harmless and no longer needed. **Wire change:** `q` is required in `public.eql_v4_json_search`, and a document written by the earlier build of this unreleased family is refused by the domain CHECK. - **The token-chain walk reads the document once.** `json_tree_subtree`, `json_tree_subtree_at`, `json_tree_children` and `json_tree_child_count` computed each parent tag inside a scan of `sv`, once per (node, ordinal, entry); a 244-entry subtree took ten seconds. The walk now builds one map of the document's entries keyed by parent tag and looks each computed tag up in it, so a level costs its fan-out in SHA-256 evaluations and the document is never scanned per node. The same subtree walks out in 23 ms; `children` and `child_count` in 1.6 ms. The walk is also bounded by the document's size, so a stored document whose unauthenticated chain fields claim a huge ordinal, or loop back to an ancestor, can no longer keep a query running until `statement_timeout`. - **A walk has a budget, and past it returns the sealed root.** `json_tree_subtree(doc, token, budget DEFAULT 64)` and `json_tree_subtree_at(doc, token, selector, budget DEFAULT 64)` stop once the subtree exceeds `budget` entries and return the root entry alone, with `d = 0`, when the document's root is sealed: the root's ciphertext is the whole document at about the plaintext's size, where the walk returns some four hundred characters per entry and pays the fan-out in hashes per node, so past a dozen or so entries the whole document is cheaper on both sides. The client opens the root and takes the walked path over it, and gets what the walked rows open to. The rows are opened through the key that keyed the walk, which carries the path it was derived from and knows from its gates which walk the rows came from: `key.open(rows)` in `stack-encrypt`, `JsonTreeSubtree::from_key(&key, rows)` in `eql-bindings`. Rows cannot be opened without it, so a document that grows past the budget does not break a caller. A budget of `0` never falls back; a token that names no node still yields no row; a document whose root is not sealed is always walked out. diff --git a/AGENTS.md b/AGENTS.md index a482387cc..951da3d93 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -691,6 +691,14 @@ monorepo, which is where the silent failures are. `jsonb @>` over the plaintexts, is `packages/eql/tests/encryption/tests/json_tree.rs`, run by `mise run test:encryption:postgres`. The design and its implementation notes are `docs/plans/2026-10-17-json-tree-index.md`. + In v4 it carries v3's names for the searchable JSON document: the target + `JsonSearch` stores `public.eql_v4_json_search`, queried with + `eql_v4.query_json` and `eql_v4.query_json_leaf`, with entries + `public.eql_v4_json_entry`; `json_tree` names the implementation only (the + SQL directory, the Rust structs, the SQL functions). The v3 SteVec SQL + that owns those names in v3 (`src/v3/json/` and the generated + `json_entry_` cross files) is `-- ONLY EQL VERSION: 3`, so no bundle + carries both. A change to the derivations on one side (a PAE domain, the header layout, the projection shape) is a change to the other: the Postgres suite is what holds them together. diff --git a/docs/plans/2026-10-17-json-tree-index.md b/docs/plans/2026-10-17-json-tree-index.md index 9f8065607..61eb02ff4 100644 --- a/docs/plans/2026-10-17-json-tree-index.md +++ b/docs/plans/2026-10-17-json-tree-index.md @@ -20,7 +20,7 @@ Three changes to the v3 SteVec design, taken together: Everything here works on standard PostgreSQL, as a non-superuser, with no extension: the SQL uses `sha256(bytea)` (core since 11), `get_byte` / `set_byte`, recursive CTEs and the `jsonb_path_ops` GIN operator class. -**It ships as a new domain type in EQL v4, not as the v4 SteVec (decided 2026-10-18).** The scheme gets its own column domain, query domain and entry domain beside `public.eql_v4_json` and whatever v4 port of the v3 SteVec `json_search` type may follow, and it shares no wire shape or SQL with them. Two reasons. It must not block the v4 release: a new domain is additive and can land in any 4.x. And this is what the domain-per-scheme design of EQL v3, carried into v4, was for: a new encryption scheme is a new domain type, with its own CHECK, operators and functions, and never a new EQL version. The domain names below use the placeholder `eql_v4_json_tree`; the final name is a Phase 4 decision. +**It ships as a new domain type in EQL v4, not as the v4 SteVec (decided 2026-10-18).** The scheme gets its own column domain, query domain and entry domain beside `public.eql_v4_json` and whatever v4 port of the v3 SteVec `json_search` type may follow, and it shares no wire shape or SQL with them. Two reasons. It must not block the v4 release: a new domain is additive and can land in any 4.x. And this is what the domain-per-scheme design of EQL v3, carried into v4, was for: a new encryption scheme is a new domain type, with its own CHECK, operators and functions, and never a new EQL version. The domain names below use the placeholder `eql_v4_json_tree`; the final name is a Phase 4 decision. (Taken on 2026-10-25: the domains carry the names EQL v3 gives its searchable JSON document, `public.eql_v4_json_search`, `public.eql_v4_json_entry`, `eql_v4.query_json` and `eql_v4.query_json_leaf`, with the target `JsonSearch`; see "Where it is" under the implementation notes. Later sections keep the placeholder names they were written with.) ## Where we are @@ -369,7 +369,7 @@ Each phase is a PR against `next`, where #1209 landed. The index, the SQL and the live oracle landed in the pull request stacked on this document; what follows records where the code is and where it departs from the design above, so the two can be read against each other. Phase 0 was folded into Phase 2, and Phases 3 and 4 remain open. -**Where it is.** The engine half is `packages/stack-encrypt/src/json/`: `Json` is the index (`Index`, term `JsonTerms`), `target::json_document` the sealing operation, `JsonDocument` the stored term and an `EncryptFrom` / `DecryptInto` target, `JsonQuery` the needle for `Contains`, `ContainedBy` and `AtPath`, `JsonLeafQuery` the operand for the orderings at a path, and `JsonSelector` / `JsonSubtreeKey` the extraction keys, derived through `KeysetCipher::json_selector` and `json_subtree_key`. `IndexSpec::Json(JsonOptions)` is the data form, with the key `"json"` and `{"json": {"sealed_root": …}}` as its wire form; a data plan refuses it as an output (`Reason::JsonIndexPending`) and reaches it through the `EncryptedJson` target, whose options are the index's (`TargetOptions::Json`), on a field of the `json` kind. The SQL is `packages/eql/src/v3/json_tree/`, confined to EQL v4; the EQL types are `packages/eql/crates/eql-bindings/src/v4/json_tree.rs`; the oracle is `packages/eql/tests/encryption/tests/json_tree.rs`, which stores the design's corpus, and batches of documents proptest generates (`json_tree/generated.rs`), encrypted beside their plaintexts and checks that `@>` and `<@` over the encrypted column select the rows PostgreSQL's own `jsonb` operators select over the plaintext column, on the GIN index and on a sequential scan, each plan shown in `EXPLAIN`. The pure-Rust half of that oracle is `packages/stack-encrypt/tests/json_index.rs` (`given_the_design_corpus` and the proptest properties in `given_generated_documents`), against a model of `jsonb @>` pinned to the answers in `jsonb-semantics.sql`. +**Where it is.** The engine half is `packages/stack-encrypt/src/json/`: `Json` is the index (`Index`, term `JsonTerms`), `target::json_document` the sealing operation, `JsonDocument` the stored term and an `EncryptFrom` / `DecryptInto` target, `JsonQuery` the needle for `Contains`, `ContainedBy` and `AtPath`, `JsonLeafQuery` the operand for the orderings at a path, and `JsonSelector` / `JsonSubtreeKey` the extraction keys, derived through `KeysetCipher::json_selector` and `json_subtree_key`. `IndexSpec::Json(JsonOptions)` is the data form, with the key `"json"` and `{"json": {"sealed_root": …}}` as its wire form; a data plan refuses it as an output (`Reason::JsonIndexPending`) and reaches it through the `JsonSearch` target, whose options are the index's (`TargetOptions::Json`), on a field of the `json` kind. The SQL is `packages/eql/src/v3/json_tree/`, confined to EQL v4, where its domains take the names EQL v3 gives its searchable JSON document (decided 2026-10-25, replacing the placeholder `eql_v4_json_tree` names above): `public.eql_v4_json_search`, `public.eql_v4_json_entry`, `eql_v4.query_json` and `eql_v4.query_json_leaf`, with the target `JsonSearch`, while the v3 SteVec SQL is confined to EQL v3; the EQL types are `packages/eql/crates/eql-bindings/src/v4/json_tree.rs`; the oracle is `packages/eql/tests/encryption/tests/json_tree.rs`, which stores the design's corpus, and batches of documents proptest generates (`json_tree/generated.rs`), encrypted beside their plaintexts and checks that `@>` and `<@` over the encrypted column select the rows PostgreSQL's own `jsonb` operators select over the plaintext column, on the GIN index and on a sequential scan, each plan shown in `EXPLAIN`. The pure-Rust half of that oracle is `packages/stack-encrypt/tests/json_index.rs` (`given_the_design_corpus` and the proptest properties in `given_generated_documents`), against a model of `jsonb @>` pinned to the answers in `jsonb-semantics.sql`. **Decisions taken by the code.** Of the eleven listed below: 1, hex for every sixteen-byte field and base64 for `c` and `h`, pending Phase 3 (taken on 2026-10-23, Phase 3d: base64 for every field the SQL reads, base85 for `c`); 2, leaf ordering terms bound to the column only, numbers as the `f64` they round to and strings as text, under the scalar `Ope` index's derivation; 3, wildcard arrays only; 4, group tags on, and only on container array elements; 5, the child count is in every container node's ciphertext, and a truncated subtree is `OpenedType`, not a shorter array; 7, plain SHA-256; 8, the option is the boolean `sealed_root`, on by default, recorded in the header's flags; 9, the projection is built from the tags server-side (`eql_v4.json_tree_query`) and `groups_ok` was not needed; 10, position-keyed parent tags on, the ordinal stored as `o`, object members ranked under the parent's token; 11, keyed: in this first form a child's token was `PRF(json/token-step ‖ column, PAE(t(parent), 0x01, key))` (`0x02, LE64(index)` for an element) and an object member's rank `PRF(json/rank ‖ column, PAE(t(parent), key))`, each under its own PAE domain, so a token's holder could neither test key names against the child tokens the walk unwraps nor rank guessed names against the ordinals. Decision 6 is out of scope, as proposed. (Both steps were first hashed unkeyed, as the token chain section above writes them, and were corrected before any release.) diff --git a/languages/golang/CHANGELOG.md b/languages/golang/CHANGELOG.md index 7d51d6e8b..c589a633e 100644 --- a/languages/golang/CHANGELOG.md +++ b/languages/golang/CHANGELOG.md @@ -13,11 +13,13 @@ listed under **Breaking**. Releases are tagged `languages/golang/vX.Y.Z`. ### Added - **The EQL types for an encrypted JSON document** (#1348): - `eql.EncryptedJSON` (`public.eql_v4_json_tree`), its containment needle - `eql.EncryptedJSONQuery`, its ordering operand - `eql.EncryptedJSONLeafQuery` and its entry `eql.EncryptedJSONEntry`, - generated from the EQL catalog. The engine stores one from the - module's own plans; `stashgen` does not generate a field of it yet. + `eql.JSONSearch` (`public.eql_v4_json_search`), its containment needle + `eql.JSONSearchQuery`, its ordering operand + `eql.JSONSearchLeafQuery` and its entry `eql.JSONEntry`, + generated from the EQL catalog. The names are the ones EQL v3 gives its + searchable JSON document; `eql.JSON` stays the storage-only type. The + engine stores one from the module's own plans; `stashgen` does not + generate a field of it yet. - **Decimals, dates and timestamps take an order index (#1236).** An `encrypt.Decimal`, `encrypt.Date` or `time.Time` field takes `ore` and @@ -49,6 +51,12 @@ listed under **Breaking**. Releases are tagged `languages/golang/vX.Y.Z`. ### Breaking +- **`eql.SteVecDocument` and `eql.SteVecQuery` are gone.** They named + EQL v4's SteVec domains, which the engine never produced and the v4 + bundle no longer carries: in v4, `public.eql_v4_json_search` is the + JSON tree document, `eql.JSONSearch`, and `eql.Types` lists it in + their place. + - **An opaque struct seals as a storage blob, and reads the JSON it used to (#1347).** `stash:"context=...,opaque"` now seals the struct as one blob in vitaminc's storage profile, version 1 (`vcffi.MarshalStorage`), diff --git a/packages/eql/AGENTS.md b/packages/eql/AGENTS.md index 6477cdded..05e5bc03d 100644 --- a/packages/eql/AGENTS.md +++ b/packages/eql/AGENTS.md @@ -60,7 +60,7 @@ Where the two versions' SQL must differ, the source holds both forms in **versio #### Files confined to one EQL version -A source file belongs to every version unless it carries the line `-- ONLY EQL VERSION: N` (the payload version, not the schema name, which the source may not write). Two surfaces do today. The generated range surface: `src/v3/scalars//query__range_types.sql` and `query__range_functions.sql` hold the range query domains (`eql_v4.query__range_ore` / `_range_ope`), their `lo` / `hi` accessors and `eql_v4.between(column, range)` (ADR-0009 decision 7). And the hand-written JSON tree family under `src/v3/json_tree/` (`types.sql`, `functions.sql`, `operators.sql`, `blockers.sql`): `public.eql_v4_json_tree`, `public.eql_v4_json_tree_entry`, `eql_v4.query_json_tree` and `eql_v4.query_json_tree_leaf`, the document Stack Encrypt's JSON index writes, a new family beside `public.eql_v4_json` and not a port of the SteVec surface (design: `docs/plans/2026-10-17-json-tree-index.md` at the repository root; bindings: `crates/eql-bindings/src/v4/json_tree.rs`, hand-written like the SteVec structs; the family is not in the catalog yet). Both are `-- ONLY EQL VERSION: 4`. EQL v3 is what cipherstash-client writes, nothing there produces a range or a JSON tree document, and its SQL must not change. `tasks/build.sh` runs the install order through `eql-codegen select <3|4>` before rendering each version, `eql-codegen render` refuses to render a confined file as the other version, `verify_installer_complete.sh` filters its disk side the same way with its own `grep`, the symbol-order checker runs on the v4 order too (on a copy mapped back to `eql_v3` names), the Doxygen filter reads a confined file from its own version's rendered tree, and `docs:validate:documented-sql` runs the whole v4 install order (`build/render/eql_v4/deps-ordered.txt`) in one rolled-back transaction, so every statement of the confined files runs after what it requires, besides running each file of the v3 render on its own. `eql-codegen order` still orders every file, so `src/deps-ordered-v3.txt` lists the v4-only files; the per-version lists are `build/render/eql_v/deps-ordered.txt`. +A source file belongs to every version unless it carries the line `-- ONLY EQL VERSION: N` (the payload version, not the schema name, which the source may not write). Three surfaces do today. The generated range surface: `src/v3/scalars//query__range_types.sql` and `query__range_functions.sql` hold the range query domains (`eql_v4.query__range_ore` / `_range_ope`), their `lo` / `hi` accessors and `eql_v4.between(column, range)` (ADR-0009 decision 7). And the hand-written JSON tree family under `src/v3/json_tree/` (`types.sql`, `functions.sql`, `operators.sql`, `blockers.sql`): `public.eql_v4_json_search`, `public.eql_v4_json_entry`, `eql_v4.query_json` and `eql_v4.query_json_leaf`, the document Stack Encrypt's JSON index writes, under the names v3 gives its searchable JSON document but not a port of the SteVec surface (design: `docs/plans/2026-10-17-json-tree-index.md` at the repository root; bindings: `crates/eql-bindings/src/v4/json_tree.rs`, hand-written like the SteVec structs; catalog: `eql_domains::DOCUMENT_FAMILIES`, whose stored type is the target `JsonSearch`). Both are `-- ONLY EQL VERSION: 4`. The third surface goes the other way: the SteVec SQL under `src/v3/json/` and the generated `json_entry_` cross files under `src/v3/scalars//` are `-- ONLY EQL VERSION: 3`, because in v4 their domain names belong to the JSON tree family. EQL v3 is what cipherstash-client writes, nothing there produces a range or a JSON tree document, and its SQL must not change: the v3 bundle differs from an unconfined build only by those directive comment lines. `tasks/build.sh` runs the install order through `eql-codegen select <3|4>` before rendering each version, `eql-codegen render` refuses to render a confined file as the other version, `verify_installer_complete.sh` filters its disk side the same way with its own `grep`, the symbol-order checker runs on the v4 order too (on a copy mapped back to `eql_v3` names), the Doxygen filter reads a confined file from its own version's rendered tree, and `docs:validate:documented-sql` runs the whole v4 install order (`build/render/eql_v4/deps-ordered.txt`) in one rolled-back transaction, so every statement of the confined files runs after what it requires, besides running each file of the v3 render on its own. `eql-codegen order` still orders every file, so `src/deps-ordered-v3.txt` lists the files confined to either version; the per-version lists are `build/render/eql_v/deps-ordered.txt`. Anything that needs the source as real SQL reads the rendered tree, not `src/`: `docs:validate:documented-sql` runs psql over `build/render/eql_v3/src`, and the Doxygen input filter (`tasks/docs/doxygen-filter.sh`) reads each file's rendered copy from `build/render/eql_v3/`. A new tool that parses `src/v3` must do the same, and depend on `build`; do not substitute the placeholders a second way. @@ -94,7 +94,7 @@ This is the **Encrypt Query Language (EQL)** - a PostgreSQL extension for search Adding a scalar encrypted-domain type is one row in the Rust catalog `eql-domains::CATALOG` (`crates/eql-domains/src/lib.rs`): a `DomainFamily` giving the type `name` (e.g. `bigint`), its `ScalarKind` (the `kind` field), the `Domain`s mapping each generated (bare) domain name to its fixed index `Term`s (`eq => [Hm]`, `ord`/`ord_ore => [Ore]`), and the `Fixture` value list. Term capabilities are fixed in the `Term` enum's `impl` methods (with unit tests): `Hm` provides equality, and `Ore` provides equality plus ordering. There is no TOML manifest and no Python — the catalog is the source of truth, validated by the compiler (an undefined term or unknown scalar is a compile error) plus catalog `#[test]`s. `mise run build` runs `cargo run -p eql-codegen`, which regenerates the scalar SQL surface into `src/v3/scalars//` from `CATALOG` at the start of every build; that surface includes supported comparison wrappers plus blockers for native `json` operators that would otherwise be reachable through domain fallback. `cargo run -p eql-codegen` regenerates every type at once (the same call `mise run build` uses; there is no per-type codegen task). The generated `*_types.sql` / `*_functions.sql` / `*_operators.sql` / `*_aggregates.sql` files are **committed in place** under `src/v3/scalars//` and drift-gated by `mise run codegen:parity` (regenerate in place + `git diff` + untracked check — the same regenerate-and-diff pattern `types:check` uses for the committed bindings). They are still machine-generated: change the catalog and rebuild, never hand-edit (CI fails on drift). The per-type plaintext fixture lists the SQLx matrix consumes are **not** a generated file — they are materialised from each `CATALOG` row at compile time as `eql_domains::INT4_VALUES` / `INT2_VALUES` (the `int_values!` macro) and read directly by `ScalarType::FIXTURE_VALUES`; a Rust source of truth no longer round-trips through a committed generated `.rs`. Generated SQL carries a `-- AUTOMATICALLY GENERATED FILE` header (the project-wide marker `docs:validate` greps on); change the catalog and rebuild, never hand-edit. Hand-written SQL beyond the fixed surface goes in `src/v3/scalars//_extensions.sql` with no auto-generated header and explicit `-- REQUIRE:` edges — that file IS committed. `json`'s **SteVec** domains never enter this pipeline, but its **scalar storage domain does**: `generate_all` iterates `eql_domains::families_with_scalar_domains()` and renders each family's scalar domains (so `public.eql_v3_json` is generated into `src/v3/scalars/json/`), while `list-types` and the SQLx matrix still iterate `eql_domains::scalar_families()`, which excludes the whole non-scalar `json` family — so `json` stays out of the operator/coverage matrix by construction (a storage-only domain carries no operator matrix), even though its ciphertext-only storage domain now has a generated SQL surface. -**EQL v4 bindings:** the generator writes the scalar domains a second time into `crates/eql-bindings/src/v4/` (`eql_v4_*` names, `v: 4` through `SchemaVersion<4>`, an `inventory.rs`), plus the Stack Encrypt target table `v4/targets.rs`. **Only v4 structs derive Stack Encrypt's `EncryptFrom`/`DecryptInto`**: Stack Encrypt's output is not byte-for-byte compatible with cipherstash-client's, so `encryption_gap` refuses every v3 domain and a codegen test asserts no v3 struct derives either. v4 also has one range query struct per ordered family and order term (`IntegerRangeOreQuery`, `TextRangeOpeQuery`, …: each bound the single-bound domain's `ob` or `op` value, typed as that twin's term newtype, in a `{v, i, lo, hi}` envelope that Stack Encrypt will need to build once the ordered types are producible), listed by `v4::all_range_query()`; v3 has none, matching the SQL. v4 has no SteVec bindings, payload enums, TypeScript export or published JSON Schema yet. The hand-written `v4/mod.rs` re-exports v3's term newtypes and `DomainType`, so a change to a term's validation in `v3/terms.rs` changes v4 too; the version boundary is the envelope's `SchemaVersion` (one generic type in `lib.rs`; `v3::SchemaVersion` and `v4::SchemaVersion` are `SchemaVersion<3>` and `SchemaVersion<4>`). +**EQL v4 bindings:** the generator writes the scalar domains a second time into `crates/eql-bindings/src/v4/` (`eql_v4_*` names, `v: 4` through `SchemaVersion<4>`, an `inventory.rs`), plus the Stack Encrypt target table `v4/targets.rs`. **Only v4 structs derive Stack Encrypt's `EncryptFrom`/`DecryptInto`**: Stack Encrypt's output is not byte-for-byte compatible with cipherstash-client's, so `encryption_gap` refuses every v3 domain and a codegen test asserts no v3 struct derives either. v4 also has one range query struct per ordered family and order term (`IntegerRangeOreQuery`, `TextRangeOpeQuery`, …: each bound the single-bound domain's `ob` or `op` value, typed as that twin's term newtype, in a `{v, i, lo, hi}` envelope that Stack Encrypt will need to build once the ordered types are producible), listed by `v4::all_range_query()`; v3 has none, matching the SQL. v4 has no SteVec bindings or SQL (its searchable JSON document is the JSON tree family, the target table's `JsonSearch` row), and no payload enums, TypeScript export or published JSON Schema yet. The hand-written `v4/mod.rs` re-exports v3's term newtypes and `DomainType`, so a change to a term's validation in `v3/terms.rs` changes v4 too; the version boundary is the envelope's `SchemaVersion` (one generic type in `lib.rs`; `v3::SchemaVersion` and `v4::SchemaVersion` are `SchemaVersion<3>` and `SchemaVersion<4>`). The same generator also emits the **Rust payload bindings** under `crates/eql-bindings/src/v3/.rs` (structs + `DomainType` impls) and the `inventory.rs` `all()` list, from the same `CATALOG` — committed with a `// @generated` header (the bindings must exist on a clean clone because `ts-rs`/`schemars` derive the committed TypeScript/JSON Schema off them; the SQL surface is now committed too, so both generated targets are handled consistently — committed in place, drift-gated by regenerate-and-`git diff`). The hand-written `DomainType` trait, the shared newtypes (`SchemaVersion`/`Identifier`/`Ciphertext`/`Hmac256`/`OreBlock256`/`BloomFilter`), the `PhantomData` plumbing, and the architectural module doc (including the non-derivable float-NaN and bool storage-only caveats) stay hand-written in `crates/eql-bindings/src/v3/{mod,domain_type,terms}.rs`. Generated structs carry a catalog-derived struct doc — a summary line (`` `eql_v3.` — . ``) plus a detail line listing the supported operators and required payload keys, all derived from data the catalog already holds (the capability label, `Term::operators_for_terms`, and `ENVELOPE_KEYS` ++ `Term::term_json_keys` — see `struct_doc_lines` in `crates/eql-codegen/src/bindings.rs`). The required-key list makes structural distinctions visible — e.g. `text_ord` lists `` `v` `i` `c` `hm` `ob` `` (dual-term) versus an integer `integer_ord`'s `` `v` `i` `c` `ob` ``. There are **no per-field docs**: per-field/term semantics live on the shared term newtypes (`terms.rs`, flowing into the TS term files and JSON Schema `$defs`), and non-derivable per-family caveats (float-NaN, bool storage-only) in `mod.rs`. Free-form prose belongs at the **struct level** (a future optional catalog `doc` field emitted as extra `#[doc]` lines), never as per-field docs. JSON Schemas are emitted by **schemars 1.x** as JSON Schema 2020-12. `mise run types:generate` regenerates the Rust bindings (via `eql-codegen bindings`) then the TS/JSON; `mise run types:check` is the committed-reference drift gate — it regenerates and `git diff`s the Rust bindings (`crates/eql-bindings/src/v3` and `src/v4`), `bindings/` and `schema/`, the same regenerate-and-diff pattern `codegen:parity` now uses for the committed SQL surface. Both gates run in CI; `mise run install-hooks` wires a pre-commit hook that runs both locally whenever a commit touches the catalog, the generator, or a committed generated surface. The `json` family's **SteVec** structs (`SteVecDocument`, `SteVecEntry`, `SteVecQuery`, and `SteVecQueryEntry`) are hand-written for the same reason as their SQL — in `crates/eql-bindings/src/v3/json.rs`, symmetric with `src/v3/json/`. Their direct optional fields and `deny_unknown_fields` validation keep the Rust, TypeScript, JSON Schema, and PostgreSQL entry contracts aligned. The family's scalar storage struct **is** generated (a flat `{v,i,c}` struct, `Json`), via `render_scalar_only_bindings(&JSON)` into `crates/eql-bindings/src/v3/json_storage.rs`; the hand-written `json.rs` re-exports it (`pub use crate::v3::json_storage::Json;`) so `super::json::Json` resolves for the generated inventory/payload. Only inventory membership and ordering are catalog-driven: `inventory.rs::all()` iterates the full `CATALOG` (not `scalar_families()`), branching on `Shape` to include the three SteVec structs plus the generated `Json` alongside the other scalar structs. diff --git a/packages/eql/crates/eql-bindings/CHANGELOG.md b/packages/eql/crates/eql-bindings/CHANGELOG.md index e82026411..6b16fe42f 100644 --- a/packages/eql/crates/eql-bindings/CHANGELOG.md +++ b/packages/eql/crates/eql-bindings/CHANGELOG.md @@ -9,11 +9,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- **`EncryptedJson`, the `json_tree` document, by name** (`targets` +- **`JsonSearch`, the `json_tree` document, by name** (`targets` feature, #1348, #1349). The target table has a row for it, generated from the catalog's document family (`eql_domains::DOCUMENT_FAMILIES`): produced from the `json` kind, carrying the `json` index, with - `EncryptedJsonQuery` (`eql_v4.query_json_tree`) as its needle. + `JsonSearchQuery` (`eql_v4.query_json`) as its needle. `encryption::targets::encrypt_with` takes the field's `TargetOptions` and seals the document's JSON text through `v4::json_tree::JsonTree` under them; `decrypt` opens it back to its text. A value that is not a @@ -21,10 +21,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `TargetError::Options`, and a one-plaintext `query` of a document `TargetError::DocumentQuery` (its operands are the engine's, by JSON operator). `v4::all()` lists `JsonTreeDocument` and - `v4::all_query()` lists `JsonTreeQuery` and `JsonTreeLeafQuery`. + `v4::all_query()` lists `JsonTreeQuery` and `JsonTreeLeafQuery`. The + contract names are the ones EQL v3 gives its searchable JSON document + (`JsonSearch`, `JsonSearchQuery`, `JsonSearchLeafQuery`, `JsonEntry`); + the structs keep the implementation's name, as v3's `SteVecDocument` + does. The table no longer has a `SteVecDocument` row: the SteVec SQL is + EQL v3 only, and `public.eql_v4_json_search` is the JSON tree document. - **The EQL v4 `json_tree` family's types, `v4::json_tree`** (`stack-encrypt` - feature). `JsonTreeDocument` is `public.eql_v4_json_tree`, `{v, i, h, sv, + feature). `JsonTreeDocument` is `public.eql_v4_json_search`, `{v, i, h, sv, q}`: the document's entries, one per JSON node, and `q`, its containment projection, which the server compares with a needle by native `jsonb @>`. `JsonTreeEntry` is one entry as `->` and the subtree reads return it, diff --git a/packages/stack-encrypt/CHANGELOG.md b/packages/stack-encrypt/CHANGELOG.md index 4e0f2d07f..1f3760e6c 100644 --- a/packages/stack-encrypt/CHANGELOG.md +++ b/packages/stack-encrypt/CHANGELOG.md @@ -36,8 +36,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **A JSON document through a data plan** (#1349). A plan field may declare the new `json` value kind and name a target that stores a - document, `EncryptedJson`, with the JSON index's options: - `"target": {"name": "EncryptedJson", "options": {"sealed_root": true, + document, `JsonSearch`, with the JSON index's options: + `"target": {"name": "JsonSearch", "options": {"sealed_root": true, "ordered": ["$.age"]}}`, or the bare name for the defaults. The plan resolves the options once when it is built (`FieldPlan::target_options`, `TargetOptions::Json`), so writes, queries and reads of the field run From d34aa24843c19648b0058c9f3198950750067100 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Sun, 25 Oct 2026 23:56:51 +0000 Subject: [PATCH 12/21] docs(eql): realign the JSON tree domain list after the rename The list of the four domains in src/v3/json_tree/types.sql was aligned for the `json_tree` names; with the shorter and longer v3 names the dashes and continuation lines no longer lined up. Comment only. Claude-Session: https://claude.ai/code/session_01HLuZxys1NDRmAkv3uhnuoJ --- packages/eql/src/v3/json_tree/types.sql | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/eql/src/v3/json_tree/types.sql b/packages/eql/src/v3/json_tree/types.sql index 536c3e9b8..7c874ebdc 100644 --- a/packages/eql/src/v3/json_tree/types.sql +++ b/packages/eql/src/v3/json_tree/types.sql @@ -10,13 +10,13 @@ --! Stack Encrypt's JSON index writes. It shares no wire shape and no SQL with --! the SteVec surface. Four jsonb-backed domains (none over another domain, --! so the native-jsonb firewall in blockers.sql attaches): ---! - public.{{prefix}}_json_search — storage: an EQL envelope `{v, i, h, sv, q}`. ---! - public.{{prefix}}_json_entry — one `sv` element (returned by `->` ---! and the subtree read). +--! - public.{{prefix}}_json_search — storage: an EQL envelope `{v, i, h, sv, q}`. +--! - public.{{prefix}}_json_entry — one `sv` element (returned by `->` +--! and the subtree read). --! - {{prefix}}.query_json — a containment needle: the element-nested ---! projection of the needle's selectors. +--! projection of the needle's selectors. --! - {{prefix}}.query_json_leaf — an ordering operand at a path: a ---! selector and a leaf ordering term. +--! selector and a leaf ordering term. --! --! The design is `docs/plans/2026-10-17-json-tree-index.md` --! at the repository root. The document's per-entry fields: From 43faafc1b48e60247be7680c086fb27f91da8a03 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 00:22:00 +0000 Subject: [PATCH 13/21] docs(stack-encrypt): say which target carries the json index The target descriptor's table said a type's indexes include `json` for a SteVec document. The v4 target table no longer has a SteVec row; the one type carrying the JSON index is the document family's stored type, `JsonSearch`, as eql-bindings' own table says. Claude-Session: https://claude.ai/code/session_01HLuZxys1NDRmAkv3uhnuoJ --- packages/stack-encrypt/src/dynamic/target.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/stack-encrypt/src/dynamic/target.rs b/packages/stack-encrypt/src/dynamic/target.rs index fe0c30a4d..3d5da5360 100644 --- a/packages/stack-encrypt/src/dynamic/target.rs +++ b/packages/stack-encrypt/src/dynamic/target.rs @@ -51,7 +51,7 @@ use crate::{KeysetCipher, Label, Pending, StackCipher}; /// | `suffix` | string: the query-capability suffix, `Eq`; empty for a storage-only type | /// | `plaintext` | string or null: the [`ValueKind`] name the type is produced from — the same names a field's `"type"` key uses — or null while unspecified | /// | `sql_domain` | string: the stored value's PostgreSQL domain | -/// | `indexes` | list of strings: the indexes the type carries, by `IndexSpec::key()`, plus `json` for a SteVec document | +/// | `indexes` | list of strings: the indexes the type carries, by `IndexSpec::key()`: `eq`, `match`, `ore`, `ope`, and `json` for a document family's stored type (`JsonSearch`) | /// | `query` | string or null: the query twin's type name | /// | `query_sql_domain` | string or null: the query twin's PostgreSQL domain | /// | `producible` | bool: whether the engine produces the type today | From b9444193ac68793fc8e8f51e6b51a7b658afe4c7 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 20:55:24 +1100 Subject: [PATCH 14/21] test(eql): compare the data plan's stored document with the typed one as sets A sealed document now stores its entries in a random order (root first) and its containment selector set shuffled, so two seals of one document differ in order. The parity test compared entries by position and q by bytes, which only held while both seals walked the document the same way. It now compares the root entry, the entries as a sorted multiset of (selector, ordering term, parent tag's place), and the two selector sets through JsonSelectorSet's set equality. Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- .../encryption/tests/json_tree_data_plan.rs | 58 ++++++++++++++----- 1 file changed, 45 insertions(+), 13 deletions(-) diff --git a/packages/eql/tests/encryption/tests/json_tree_data_plan.rs b/packages/eql/tests/encryption/tests/json_tree_data_plan.rs index c564093af..f10a6c3d2 100644 --- a/packages/eql/tests/encryption/tests/json_tree_data_plan.rs +++ b/packages/eql/tests/encryption/tests/json_tree_data_plan.rs @@ -11,7 +11,7 @@ //! - a batch of documents seals in one key request and opens to the same //! values, scalar roots, arrays and a JSON `null` included; //! - the data plan and `JsonTree::new(options)` write the same search terms -//! (selectors, ordering terms, the containment projection) for the same +//! (selectors, ordering terms, the containment selector set) for the same //! document under the same column, and each opens what the other sealed; //! - text that is not a document is refused before any key is requested, //! naming the field and a code, never a byte of the text. @@ -27,7 +27,7 @@ use serde_json::{json, Value}; use stack_encrypt::dynamic::{ self, record, Scope, TargetDescriptor, TargetError, TargetResolver, ValueFrame, }; -use stack_encrypt::json::JsonOptions; +use stack_encrypt::json::{JsonDocument, JsonOptions}; use stack_encrypt::target::{ExpectedContext, Layout}; use stack_encrypt::{CipherText, Encrypt, KeysetCipher, Label, NonEmpty, Pending, StackCipher}; @@ -188,6 +188,33 @@ mod serde_bytes_like { } } +/// The JSON index's own document under the EQL envelope, whose selector +/// set compares as a set. +fn index_document(document: &JsonTreeDocument) -> JsonDocument { + let wire = serde_json::to_value(document).unwrap(); + serde_json::from_value(json!({"h": wire["h"], "sv": wire["sv"], "q": wire["q"]})).unwrap() +} + +/// What each stored entry shows a query, as a sorted list: its selector, +/// its ordering term and whether it carries a parent tag. Entries are +/// stored in a random order, so two seals of one document compare as +/// multisets. +fn search_terms(document: &JsonTreeDocument) -> Vec<(String, String, bool)> { + let mut terms: Vec<_> = document + .sv + .iter() + .map(|entry| { + ( + serde_json::to_string(&entry.s).unwrap(), + serde_json::to_string(&entry.op).unwrap(), + entry.p.is_some(), + ) + }) + .collect(); + terms.sort(); + terms +} + fn column() -> NonEmpty { Identifier::for_column("users", "profile").unwrap() } @@ -229,10 +256,11 @@ async fn a_batch_of_documents_round_trips_in_one_request() { } /// The data plan and the typed layout write the same search terms for the -/// same document under the same column: every entry's selector, self and -/// parent tags' places and ordering term, and the containment projection. -/// Only the ciphertexts, the data key and the tags salted by it differ. -/// And each opens what the other sealed. +/// same document under the same column: the root's entry first, then every +/// entry's selector, parent tag's place and ordering term, and the +/// containment selector set, compared as sets because each seal draws its +/// own storage order. Only the ciphertexts, the data key, the tags salted +/// by it and the order differ. And each opens what the other sealed. #[tokio::test] async fn the_data_plan_and_the_typed_layout_agree() { let (cipher, _) = common::cipher().await; @@ -255,13 +283,17 @@ async fn the_data_plan_and_the_typed_layout_agree() { .await .unwrap(); assert_eq!(data.i, typed.i, "{text}: the column"); - assert_eq!(data.q, typed.q, "{text}: the containment projection"); - assert_eq!(data.sv.len(), typed.sv.len(), "{text}: one entry per node"); - for (a, b) in data.sv.iter().zip(&typed.sv) { - assert_eq!(a.s, b.s, "{text}: an entry's selector"); - assert_eq!(a.op, b.op, "{text}: an entry's ordering term"); - assert_eq!(a.p.is_some(), b.p.is_some(), "{text}: a parent tag's place"); - } + assert_eq!( + index_document(&data).selector_set(), + index_document(&typed).selector_set(), + "{text}: the containment selector set" + ); + assert_eq!(data.sv[0].s, typed.sv[0].s, "{text}: the root first"); + assert_eq!( + search_terms(&data), + search_terms(&typed), + "{text}: one entry per node, each with its selector, ordering term and parent tag's place" + ); // The typed path opens the data plan's document... let opened: Value = cipher From c0cedd519f9afbac90e77a8d47586f1dc4233816 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 20:55:25 +1100 Subject: [PATCH 15/21] docs(stack-encrypt): name JsonSelectorSet in the fuzz:json-wire task The task's description still listed JsonProjection, the selector set's former name, among the decoders the target fuzzes. Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- packages/stack-encrypt/tasks.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/stack-encrypt/tasks.toml b/packages/stack-encrypt/tasks.toml index bb337a20c..02913222b 100644 --- a/packages/stack-encrypt/tasks.toml +++ b/packages/stack-encrypt/tasks.toml @@ -68,7 +68,7 @@ run = "mise x --env test -- cargo +nightly fuzz run plan_build --sanitizer none # aarch64 and x86_64) and the header's frozen layout must never panic, and an # accepted document or entry must survive a round trip through its wire form. ["fuzz:json-wire"] -description = "Fuzz stack-encrypt's JSON index wire decoders: JsonDocument, JsonNode, JsonProjection, JsonLeafQuery and JsonHeader (libFuzzer, nightly, 60s default)" +description = "Fuzz stack-encrypt's JSON index wire decoders: JsonDocument, JsonNode, JsonSelectorSet, JsonLeafQuery and JsonHeader (libFuzzer, nightly, 60s default)" dir = "{{config_root}}/packages/stack-encrypt" run = "mise x --env test -- cargo +nightly fuzz run json_wire_decode --sanitizer none --target $(rustc -vV | sed -n 's/^host: //p') -- -max_total_time=60" From c0316443d364a12ff77a3c846679d81e244e8ed2 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 21:24:13 +1100 Subject: [PATCH 16/21] docs(stack-encrypt): say which heap blocks parse_text leaves clean The parser's rustdoc and the changelog said no heap block it frees holds plaintext. That holds for strings and array buffers, but an object's members go into serde_json's BTreeMap, which splits a full node by copying half its slots and does not clear the ones it leaves: a number or boolean member of an object with twelve or more members can stay in a freed node. Strings move by pointer, so their bytes never sit there. Both now say so, and that numbers and booleans are wiped best-effort. Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- packages/stack-encrypt/CHANGELOG.md | 7 +++++-- packages/stack-encrypt/src/json/text.rs | 19 ++++++++++++++----- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/packages/stack-encrypt/CHANGELOG.md b/packages/stack-encrypt/CHANGELOG.md index 9f7f95df1..9c0d3db86 100644 --- a/packages/stack-encrypt/CHANGELOG.md +++ b/packages/stack-encrypt/CHANGELOG.md @@ -69,8 +69,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 its text in a bytes leaf (`dynamic::read_json`, `dynamic::json_frame`). `json::parse_text` is the one parser, the engine's own (held to serde_json's by a differential property test and the `fuzz:json-text` - target), so no heap block it frees holds the document: it refuses an - object that names a member twice and an integer literal outside + target), so no heap block it frees holds the document's strings or an + array's buffer (a number or boolean in an object of twelve or more + members can stay in a map node it frees; those are wiped best-effort): + it refuses an object that names a member twice and an integer literal + outside `i64::MIN ..= u64::MAX` (which serde_json would read as the nearest `f64`), as well as text that is not JSON, each with a code (`JsonTextError::code`) and a position, never a byte of the text. diff --git a/packages/stack-encrypt/src/json/text.rs b/packages/stack-encrypt/src/json/text.rs index b6b5f6211..1d012451a 100644 --- a/packages/stack-encrypt/src/json/text.rs +++ b/packages/stack-encrypt/src/json/text.rs @@ -79,11 +79,20 @@ impl JsonTextError { /// Parse a document from UTF-8 JSON text by the rules in the module docs. /// /// The parser is the engine's own, not serde_json's deserializer, so that -/// no heap block it frees holds plaintext: each string is decoded into a -/// buffer sized for it before it is read, so it never grows; a container's -/// buffer is wiped before it is replaced by a larger one; numbers are read -/// from the text in place; and on a refusal the part of the document built -/// so far is wiped ([`wipe_value`](super::wipe_value)) before it drops. +/// it leaves no string and no array buffer in a heap block it frees: each +/// string is decoded into a buffer sized for it before it is read, so it +/// never grows; an array's buffer is wiped before it is replaced by a +/// larger one; numbers are read from the text in place; and on a refusal +/// the part of the document built so far is wiped +/// ([`wipe_value`](super::wipe_value)) before it drops. +/// +/// An object's members go into serde_json's map, a `BTreeMap`, which +/// splits a full node by copying half its members to a new one and does +/// not clear the slots it leaves. Strings move by pointer, so their bytes +/// never sit in those slots, but a number or a boolean is held inline: in +/// an object of twelve or more members, a scalar member's value can stay +/// in a node the map frees. Numbers and booleans are wiped best-effort, as +/// [`wipe_value`](super::wipe_value) says. /// /// # Errors /// From f2573fea02bbc83ecf9308cb77ea7ee88798b301 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 21:24:13 +1100 Subject: [PATCH 17/21] perf(stack-encrypt): move a document's text into its frame json_frame copied the text into the Protected value and wiped the original. Taking the buffer moves it, so the plaintext exists once. Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- packages/stack-encrypt/src/dynamic/json.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/stack-encrypt/src/dynamic/json.rs b/packages/stack-encrypt/src/dynamic/json.rs index 2c5063b91..f1d56620f 100644 --- a/packages/stack-encrypt/src/dynamic/json.rs +++ b/packages/stack-encrypt/src/dynamic/json.rs @@ -87,16 +87,17 @@ pub fn read_json(value: &ValueFrame) -> Result { } /// A document as the value a `json` field opens to: its JSON text -/// ([`to_text`]), in a bytes leaf. The text is written into the frame's -/// protected buffer and the copy wiped. +/// ([`to_text`]), in a bytes leaf. The text's buffer moves into the +/// frame's protected value, so no second copy of it is made. /// /// # Errors /// /// [`crate::Error::Other`] if the frame writer refuses the bytes, which a /// document's text never is. pub fn json_frame(value: &Value) -> Result { - let text = Zeroizing::new(to_text(value).map_err(|e| crate::Error::Other(Box::new(e)))?); - ValueFrame::of(Protected::new(text.to_vec())).map_err(|e| crate::Error::Other(Box::new(e))) + let mut text = Zeroizing::new(to_text(value).map_err(|e| crate::Error::Other(Box::new(e)))?); + ValueFrame::of(Protected::new(std::mem::take(&mut *text))) + .map_err(|e| crate::Error::Other(Box::new(e))) } #[cfg(test)] From 05d19b3302cc8e7cfa72c88fbcfdaa07c53d8306 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 21:24:14 +1100 Subject: [PATCH 18/21] docs(stack-encrypt): say why a document is parsed twice and how target options are refused check_field parses a json field's document and drops it, and the resolver parses it again when it seals: the check runs before any key and needs no resolver, and a resolver takes the field's value as a frame. Options are read in the JSON index's grammar whatever the target, so options outside it are an invalid target while well-formed options on a target that takes none are target_options; the rustdoc on TargetWire now says both. Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- packages/stack-encrypt/src/dynamic/record.rs | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/packages/stack-encrypt/src/dynamic/record.rs b/packages/stack-encrypt/src/dynamic/record.rs index dd1a60b9e..451bcf03c 100644 --- a/packages/stack-encrypt/src/dynamic/record.rs +++ b/packages/stack-encrypt/src/dynamic/record.rs @@ -1410,6 +1410,14 @@ impl TryFrom for FieldSpec { /// defaults, or an object of its name and its options, which are the /// options of the index the target carries (the JSON index's options /// object, as an output spells it). +/// +/// The options are read in the JSON index's grammar whatever the target, +/// since that is the only grammar any target takes, and the target is +/// resolved after: options that are not that grammar (`{"x": 1}`) make +/// the target unreadable ([`Reason::InvalidTarget`]), and well-formed +/// options on a target that takes none (`{}` on `TextEq`) are +/// [`Reason::TargetOptions`]. Both refuse the plan before anything is +/// sealed. #[derive(Deserialize)] #[serde(untagged)] enum TargetWire { @@ -2393,7 +2401,13 @@ fn check_field(value: &ValueFrame, field: &FieldPlan) -> Result<(), Error> { } } // A document is read by the engine's rules here, before any key, so a - // check refuses what encrypt refuses, and no resolver has to. + // check refuses what encrypt refuses, and no resolver has to. The + // parse is kept to a check and the document dropped (wiped, as every + // `JsonPlaintext` is): the resolver parses it again when it seals, so + // a large document is parsed twice. Deliberate: the check runs before + // any key or keyset load and needs no resolver, and a resolver takes + // the field's value as a frame, as it does for every target, so it + // reads the text it is handed. if field.field_type == Some(ValueKind::Json) { if let Err(error) = super::read_json(value) { return Err(Error::Target(TargetError::Json { From 20f099e43aa76ef07a46aa78c1db9e0fd6a9b92a Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 21:24:14 +1100 Subject: [PATCH 19/21] test(eql,go): pin a document's column and its SQL null through the data plan Nothing opened a JsonSearch value under another column. The data plan test now seals under users/profile and opens under users/other, which is refused as a context mismatch before any key is retrieved; the Go guest test does the same and gets ErrContextMismatch. A row whose document is a frame null, or absent, under null=sql stores no value and opens as a null, beside a JSON null that round-trips as a document: the doc comment that promised this had no test. Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- languages/golang/encrypt/json_tree_test.go | 17 +++ .../encryption/tests/json_tree_data_plan.rs | 106 +++++++++++++++++- 2 files changed, 121 insertions(+), 2 deletions(-) diff --git a/languages/golang/encrypt/json_tree_test.go b/languages/golang/encrypt/json_tree_test.go index 2259291b1..dce40bcce 100644 --- a/languages/golang/encrypt/json_tree_test.go +++ b/languages/golang/encrypt/json_tree_test.go @@ -86,6 +86,23 @@ func TestJSONDocumentsRoundTripThroughTheGuest(t *testing.T) { } } +// A document sealed for users/profile does not open as another column's: +// the guest opens it under the field's own column and refuses a stored +// value whose envelope names another. +func TestJSONDocumentsDoNotOpenUnderAnotherColumn(t *testing.T) { + c := deterministicEQLClient(t) + ctx := context.Background() + sealed, err := c.DefaultKeyset().Seal(ctx, jsonPlan(), []record.Source{{"profile": []byte(jsonDocuments[0])}}) + if err != nil { + t.Fatal(err) + } + other := jsonPlan() + other.Fields[0].Identity = "other" + if _, err := c.Open(ctx, other, sealed); !errors.Is(err, encrypt.ErrContextMismatch) { + t.Fatalf("a users/profile document opened as users/other: %v", err) + } +} + // Text that is not a document is refused before anything is sealed, with // the engine's code and without the text. func TestJSONTextThatIsNotADocumentIsErrEncoding(t *testing.T) { diff --git a/packages/eql/tests/encryption/tests/json_tree_data_plan.rs b/packages/eql/tests/encryption/tests/json_tree_data_plan.rs index f10a6c3d2..648cb6f9b 100644 --- a/packages/eql/tests/encryption/tests/json_tree_data_plan.rs +++ b/packages/eql/tests/encryption/tests/json_tree_data_plan.rs @@ -328,10 +328,112 @@ async fn the_data_plan_and_the_typed_layout_agree() { } } +/// Under `"null": "sql"` a frame null, or a row with no `profile`, stores +/// no value for the field, and opens as a null; a JSON `null` beside them +/// is a document and round-trips as one. +#[tokio::test] +async fn a_sql_null_stores_no_document() { + let (cipher, _) = common::cipher().await; + let keyset = cipher.default_keyset(); + let plan = plan(); + let rows: Vec>> = vec![ + BTreeMap::from([("profile", Some(r#"{"name": "Alice"}"#))]), + BTreeMap::from([("profile", None)]), + BTreeMap::new(), + BTreeMap::from([("profile", Some("null"))]), + ]; + let sealed = record::encrypt_with(&keyset, frame(&rows), &plan, &Eql) + .unwrap() + .await + .unwrap(); + let CipherText::Sequence(stored) = &sealed else { + panic!("a batch is a sequence") + }; + let stores_profile: Vec = stored + .iter() + .map(|row| { + let CipherText::Map(fields) = row else { + panic!("a row is a map") + }; + fields.iter().any(|(name, _)| name == "profile") + }) + .collect(); + assert_eq!(stores_profile, [true, false, false, true]); + let opened = record::decrypt_with(Scope::Client(&cipher), sealed, &plan, None, &Eql) + .unwrap() + .await + .unwrap(); + let rows: Vec>> = + dynamic::frame::serde::from_slice(opened.risky_ref()).unwrap(); + let back: Vec> = rows + .iter() + .map(|row| { + row["profile"] + .as_ref() + .map(|text| serde_json::from_slice(&text.0).unwrap()) + }) + .collect(); + assert_eq!( + back, + [ + Some(json!({"name": "Alice"})), + None, + None, + Some(Value::Null) + ] + ); +} + +/// A document sealed for one column does not open as another's: the data +/// plan opens `JsonSearch` under the field's own column, and a document +/// whose envelope names another is refused before any key is retrieved. +#[tokio::test] +async fn a_document_does_not_open_under_another_column() { + let (cipher, calls) = common::cipher().await; + let keyset = cipher.default_keyset(); + let text = documents()[0].to_string(); + let sealed = record::encrypt_with( + &keyset, + frame(BTreeMap::from([("profile", text.as_str())])), + &plan(), + &Eql, + ) + .unwrap() + .await + .unwrap(); + let other = record::plan_with( + &dynamic::frame::serde::to_vec(&json!({ + "profile": { + "context": ["users", "other"], + "type": "json", + "null": "sql", + "target": { + "name": "JsonSearch", + "options": {"ordered": ["$.age", "$.visits[*].at"]} + } + } + })) + .unwrap(), + &Eql, + ) + .unwrap(); + calls.lock().unwrap().retrieve.clear(); + let refused = record::decrypt_with(Scope::Client(&cipher), sealed, &other, None, &Eql) + .unwrap() + .await; + assert!( + matches!(refused, Err(stack_encrypt::Error::ContextMismatch { .. })), + "{:?}", + refused.map(drop) + ); + assert!( + calls.lock().unwrap().retrieve.is_empty(), + "refused before any key is retrieved" + ); +} + /// Text that is not a document is refused when the record is read, before /// any key is requested, naming the field and what was wrong by its code. -/// A frame null is not a JSON null: under `"null": "sql"` it stores -/// nothing. #[tokio::test] async fn text_that_is_not_a_document_is_refused_before_any_key() { let (cipher, calls) = common::cipher().await; From 02acc73e00d7d5542ac53b6cd8f8dce2d779ac16 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 21:24:14 +1100 Subject: [PATCH 20/21] fix(go): refuse a JSON field with no target in Plan.Validate The engine refuses a json field that names no target (json_needs_target) after the plan crosses the guest. Validate already refused target options without a target; it now refuses a JSON document with no target too, and names the field before the guest is asked. Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- languages/golang/internal/record/record.go | 6 ++++++ languages/golang/internal/record/record_test.go | 4 ++++ 2 files changed, 10 insertions(+) diff --git a/languages/golang/internal/record/record.go b/languages/golang/internal/record/record.go index 8b11cfbe6..8f70a514e 100644 --- a/languages/golang/internal/record/record.go +++ b/languages/golang/internal/record/record.go @@ -332,6 +332,12 @@ func (p *Plan) Validate() error { if f.TargetOptions != nil && !f.IsTarget() { return fmt.Errorf("record: field %q has target options and no target", f.Name) } + // A JSON document is stored only through a target that stores one + // (JsonSearch); the engine refuses a json field with no target + // (json_needs_target), and saying it here names the field first. + if f.Kind == JSON && !f.IsTarget() { + return fmt.Errorf("record: field %q is a JSON document and names no target; a document is stored through an EQL type that stores one, such as JsonSearch", f.Name) + } if f.IsTarget() { if len(f.Outputs) != 0 { return fmt.Errorf("record: field %q names the EQL type %s and outputs; a field has one or the other", f.Name, f.Target) diff --git a/languages/golang/internal/record/record_test.go b/languages/golang/internal/record/record_test.go index aa0c01856..ed28f8c51 100644 --- a/languages/golang/internal/record/record_test.go +++ b/languages/golang/internal/record/record_test.go @@ -212,6 +212,10 @@ func TestWireGivesATargetItsOptions(t *testing.T) { if err := bad.Validate(); err == nil || !strings.Contains(err.Error(), "no target") { t.Fatalf("options with no target: %v", err) } + noTarget := &Plan{Context: []string{"users"}, Fields: []Field{{Name: "profile", Kind: JSON, Outputs: []Output{Ciphertext}}}} + if err := noTarget.Validate(); err == nil || !strings.Contains(err.Error(), `"profile" is a JSON document and names no target`) { + t.Fatalf("a json field with no target: %v", err) + } if !JSON.Known() { t.Fatal("json is a kind") } From c22b688b2fc04f04444bfc56c8d877835673cc47 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 21:24:14 +1100 Subject: [PATCH 21/21] docs(eql,go): name the version-confined surfaces and the test that stores documents build.sh's comment named only the range surface as confined to one EQL version; the JSON tree family (v4) and the SteVec SQL (v3) are confined too. The per-type PostgreSQL test skips JsonSearch, and its comment now names TestJSONDocumentsQueryAndReadInPostgres, which stores documents through the generated fields into an eql_v4_json_search column. Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- languages/golang/encrypt/eql_postgres_test.go | 5 +++-- packages/eql/tasks/build.sh | 7 +++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/languages/golang/encrypt/eql_postgres_test.go b/languages/golang/encrypt/eql_postgres_test.go index fcdc81d65..954b644b0 100644 --- a/languages/golang/encrypt/eql_postgres_test.go +++ b/languages/golang/encrypt/eql_postgres_test.go @@ -114,8 +114,9 @@ func TestEQLTypesRoundTripAndQueryInPostgres(t *testing.T) { continue } // A document has operators of its own, not one query per index, - // so it is not run here or counted: it is run in PostgreSQL with - // the generated fields that store it. + // so it is not run here or counted: TestJSONDocumentsQueryAndReadInPostgres + // stores documents through the generated fields (the data plan) + // into an eql_v4_json_search column and runs every operator there. if typ.Plaintext == "json" { continue } diff --git a/packages/eql/tasks/build.sh b/packages/eql/tasks/build.sh index de6edb8a8..961b6705c 100755 --- a/packages/eql/tasks/build.sh +++ b/packages/eql/tasks/build.sh @@ -91,8 +91,11 @@ mv src/deps-ordered-v3.txt.tmp src/deps-ordered-v3.txt # writes nothing unless every file renders. # # Not every file belongs to every version. A file with a `-- ONLY EQL VERSION: N` -# line belongs to version N alone: today the generated range query domains and -# `between` (query__range_*.sql, ADR-0009 decision 7) are EQL v4 only. +# line belongs to version N alone. Three surfaces carry one today: the +# generated range query domains and `between` (query__range_*.sql, ADR-0009 +# decision 7) and the JSON tree family (src/v3/json_tree/) are EQL v4 only, and +# the SteVec SQL (src/v3/json/) with the generated json_entry_ cross files is +# EQL v3 only, since in v4 its domain names belong to the JSON tree family. # `eql-codegen select` drops the files that belong to the other version from the # install order, keeping the order of the rest, and `render` refuses to render a # file as a version it does not belong to, so nothing confined to v4 can reach