From 0b001bf2395b227524579536d0914377b9c2a154 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 00:01:05 +1100 Subject: [PATCH 01/22] test(stack-encrypt): compile the float round-trip test after the token-list merge The test module #1366 added to `tests/json_index.rs` was written against the gated walk: it called `subtree_nodes(doc, token)`, which the token-list read (#1363) replaced with `read_rows(doc, key)`, and opened an extracted element with `decrypt_as::`, which has no implementation (an extracted entry opens through `JsonExtracted`, which checks its selector first). Merged on top of #1363 the file no longer compiled, and the `tests-crates` workflow is red on `next` at ebdb5de0a. Both calls now use the helpers the rest of the file uses. The test's assertions are unchanged. Refs #1366, #1363. Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- packages/stack-encrypt/tests/json_index.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/stack-encrypt/tests/json_index.rs b/packages/stack-encrypt/tests/json_index.rs index c893bc0c9..0ca1634d9 100644 --- a/packages/stack-encrypt/tests/json_index.rs +++ b/packages/stack-encrypt/tests/json_index.rs @@ -2157,7 +2157,7 @@ mod given_a_float_at_full_precision { let key = subtree_key(cipher, "$.list", JsonOptions::default()) .await .unwrap(); - let rows = subtree_nodes(&sealed, key.token_bytes()); + let rows = read_rows(&sealed, &key); let element = sealed.node(2).unwrap(); vec![ ( @@ -2171,7 +2171,10 @@ mod given_a_float_at_full_precision { ( "the extracted element, as `->` returns it", cipher - .decrypt_as::(element, aead()) + .decrypt_as::( + extracted(cipher, "$.list[*]", element).await, + aead(), + ) .await .unwrap(), ), From 8faa096dc7e039b03c215f5c5d76a3e1d88a7249 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 11:55:56 +1100 Subject: [PATCH 02/22] feat(stack-encrypt): cascade a JSON node's children's AEAD tags into its AAD MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A json_tree read used to be checked by position: the rows under the node a token named had to sit at consecutive slots in depth-first order, and each entry's AAD bound its slot. That rule is sound only while the stored order is the depth-first order, and the stored order is what reveals each row's tree (#1357). To store the entries in any other order, the client needs an integrity check that does not read the slot as a position. Each entry's AAD now binds the digest of its children's AEAD tags, SHA-256("k" ‖ tag(child_0) ‖ tag(child_1) ‖ …) in ordinal order, under the new domain json/entry/v2. A node's tag covers its AAD, so it covers its children's tags, which cover theirs: a node commits to its whole subtree. Sealing runs from the leaves up. A leaf, an empty container and the sealed root (whose members chain from nothing) open under the empty digest. Nothing stored changes: the AAD is not stored and the tag is already the last sixteen bytes of each ciphertext. Opening a read places its rows by the chain, as the server finds them: the node a token names by its self tag, a node's children by its parent tags at ordinals 0, 1, 2, … Each node then opens under the digest of the children the tags gave it, so a row swapped, dropped, added, reordered or moved under another parent is refused by the cipher (Error::Aead). The client still refuses, as JsonWrongEntry, a row placed twice, a row no read reaches, a reported depth the chain does not give, a selector or label that is not the path's, a child under the wrong rule, the sealed root anywhere but at `$`, and a container whose child count disagrees. The consecutive-position rule is gone. Opening a whole document whose root is open walks the chain the same way, so it derives the root's token; for now it takes the keyset from the cipher's cache. `->` of a container with children no longer opens alone (its AAD covers tags `->` does not return) and is refused by the cipher where it was refused by kind. The new code lives in json/cascade.rs (the digest and the seal order) and derive.rs (`place` and `Placing`). The hostile-row tests that asserted the position check's errors are rewritten in a later commit. Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- packages/stack-encrypt/src/cipher.rs | 10 + packages/stack-encrypt/src/json/cascade.rs | 184 ++++++ packages/stack-encrypt/src/json/derive.rs | 716 ++++++++++++--------- packages/stack-encrypt/src/json/mod.rs | 1 + 4 files changed, 624 insertions(+), 287 deletions(-) create mode 100644 packages/stack-encrypt/src/json/cascade.rs diff --git a/packages/stack-encrypt/src/cipher.rs b/packages/stack-encrypt/src/cipher.rs index 6c469e8ed..4609ad412 100644 --- a/packages/stack-encrypt/src/cipher.rs +++ b/packages/stack-encrypt/src/cipher.rs @@ -592,6 +592,16 @@ impl StackCipher { &self.kms } + /// The loaded state of the keyset with `id`, if this cipher holds it: + /// what opening a JSON document with its root open derives the root's + /// token with. + pub(crate) fn loaded_keyset(&self, id: Uuid) -> Option> { + match self.keysets().get(&IdentifiedBy::Uuid(id)) { + Lookup::Hit(state) => Some(state), + Lookup::Stale(_) | Lookup::Miss(_) => None, + } + } + fn keysets(&self) -> std::sync::MutexGuard<'_, KeysetCache> { // The cache holds no invariant a panic mid-update could break (an // insert is two map writes, and a stale name entry only points at diff --git a/packages/stack-encrypt/src/json/cascade.rs b/packages/stack-encrypt/src/json/cascade.rs new file mode 100644 index 000000000..b56337761 --- /dev/null +++ b/packages/stack-encrypt/src/json/cascade.rs @@ -0,0 +1,184 @@ +//! The cascade: how a JSON document's ciphertexts commit to the tree. +//! +//! Every entry is sealed under an AAD that binds, beside its keyset, its +//! column, its key tag, its selector and its stored slot, the +//! [`ChildrenDigest`] of its children: SHA-256 over the AEAD tags of the +//! node's children, in ordinal order (an element's index, a member's keyed +//! rank). A node's AEAD tag covers its own AAD, so it covers its children's +//! tags, which cover theirs: the tag of a node commits to every ciphertext +//! beneath it, in its place. Sealing therefore runs from the leaves up, and +//! opening a node needs its children's tags first. +//! +//! What that proves, for a read that returns a node and everything beneath +//! it: the rows beneath the node are exactly the ones sealed there, complete, +//! each under the parent it was sealed under and in the order it was sealed +//! in, whatever order the rows arrive in. A row swapped, dropped, added, +//! repeated, reordered or moved under another parent changes a tag on the way +//! up, and the node above the change does not open ([`Error::Aead`]). +//! +//! What it does not prove: the place in the document of the node at the top +//! of a read. That node opens under its own selector and slot, and its label +//! is inside its ciphertext, so the client knows it is a genuine node at the +//! path asked for, but not which one when several share a selector (ADR +//! 0010's allowances: another element's node at depth `0`, another row's +//! entry, withheld rows). +//! +//! Nothing stored changes: the AAD is not stored, and the tag is already the +//! last sixteen bytes of every ciphertext. +//! +//! [`Error::Aead`]: crate::Error::Aead + +use sha2::{Digest, Sha256}; + +/// The digest of a node's children's AEAD tags, bound into the node's AAD. +pub(crate) type ChildrenDigest = [u8; 32]; + +/// What the digest's input starts with, so it is never the hash of the bare +/// concatenation of the tags. +const CHILDREN_DOMAIN: &[u8] = b"k"; + +/// How many bytes of an AES-GCM ciphertext are its tag. +const AEAD_TAG_LEN: usize = 16; + +/// The AEAD tag of a sealed entry: the last sixteen bytes of its ciphertext, +/// as vitaminc lays a leaf out (`version ‖ nonce ‖ ciphertext ‖ tag`). +/// Public, like the ciphertext it ends. A ciphertext shorter than a tag +/// (which no sealing produces, and which never opens) is taken whole, so it +/// still contributes what it is to its parent's digest. +pub(crate) fn aead_tag(c: &[u8]) -> &[u8] { + &c[c.len().saturating_sub(AEAD_TAG_LEN)..] +} + +/// `SHA-256("k" ‖ tag(c₀) ‖ tag(c₁) ‖ …)` over the ciphertexts of a node's +/// children, in ordinal order. Plain SHA-256, like the chain's other hashes: +/// the tags are public and fixed-width, and nothing secret is hashed. +pub(crate) fn children_digest<'c>(children: impl IntoIterator) -> ChildrenDigest { + let mut hasher = Sha256::new(); + hasher.update(CHILDREN_DOMAIN); + for c in children { + hasher.update(aead_tag(c)); + } + hasher.finalize().into() +} + +/// The digest of no children: what a leaf, an empty container and the sealed +/// root open under. The sealed root's ciphertext is the whole document, and +/// the budget fallback returns it alone, so its AAD covers no other entry; +/// it commits to the chain's heads inside its plaintext instead. +pub(crate) fn no_children() -> ChildrenDigest { + children_digest(std::iter::empty()) +} + +/// The children of each node, as indices into `parents`, in the order a +/// digest takes them: `parents[i]` is node `i`'s parent and its ordinal under +/// it, `None` for a node that chains from nothing (the root, and the chain's +/// heads under the sealed root). +pub(crate) fn children_of( + parents: impl IntoIterator>, +) -> Vec> { + let parents: Vec> = parents.into_iter().collect(); + let mut children: Vec> = vec![Vec::new(); parents.len()]; + for (index, parent) in parents.iter().enumerate() { + if let Some((parent, ordinal)) = parent { + if let Some(list) = children.get_mut(*parent) { + list.push((*ordinal, index)); + } + } + } + children + .into_iter() + .map(|mut list| { + list.sort_unstable(); + list.into_iter().map(|(_, index)| index).collect() + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use vitaminc_aead::{Cipher, Context, LocalCipherText}; + use vitaminc_encrypt::{Aes256Cipher, Key}; + use vitaminc_protected::Protected; + + #[test] + fn the_aead_tag_is_the_last_sixteen_bytes_of_the_ciphertext() { + let c: Vec = (0..40).collect(); + assert_eq!(aead_tag(&c), &c[24..]); + assert_eq!(aead_tag(&c).len(), 16); + assert_eq!( + aead_tag(&[1, 2, 3]), + &[1, 2, 3], + "shorter than a tag: whole" + ); + assert_eq!(aead_tag(&[]), &[] as &[u8]); + } + + /// vitaminc lays a sealed leaf out as `version ‖ nonce ‖ ciphertext ‖ + /// tag`, and GCM's ciphertext is as long as its plaintext: so the last + /// sixteen bytes are the tag, and flipping any of them, which changes + /// nothing but the tag, is refused. + #[test] + fn a_sealed_leaf_ends_in_its_tag() { + let cipher = Aes256Cipher::new(&Key::from([9; 32])).unwrap(); + let aad = Context::from_encoded(b"aad"); + let empty = cipher.seal(Protected::new(Vec::new()), &aad).unwrap(); + let five = cipher + .seal(Protected::new(vec![1, 2, 3, 4, 5]), &aad) + .unwrap(); + let header = empty.as_ref().len() - AEAD_TAG_LEN; + assert_eq!(five.as_ref().len(), header + 5 + AEAD_TAG_LEN); + for at in five.as_ref().len() - AEAD_TAG_LEN..five.as_ref().len() { + let mut bytes = five.as_ref().to_vec(); + bytes[at] ^= 1; + assert!(cipher.open(LocalCipherText::from(bytes), &aad).is_err()); + } + } + + #[test] + fn the_digest_covers_each_tag_in_order() { + let a: Vec = (0..40).collect(); + let b: Vec = (100..140).collect(); + let ab = children_digest([a.as_slice(), b.as_slice()]); + let ba = children_digest([b.as_slice(), a.as_slice()]); + assert_ne!(ab, ba, "order is part of the digest"); + assert_ne!(ab, no_children(), "a container is not a leaf"); + assert_ne!(children_digest([a.as_slice()]), ab, "a dropped child"); + assert_ne!( + children_digest([a.as_slice(), b.as_slice(), a.as_slice()]), + ab, + "an added child" + ); + let mut tag_flipped = a.clone(); + tag_flipped[39] ^= 0xff; + assert_ne!( + children_digest([a.as_slice()]), + children_digest([tag_flipped.as_slice()]) + ); + // Only the tag is digested: the rest of a ciphertext is covered by + // the tag itself. + let mut body_flipped = a.clone(); + body_flipped[0] ^= 0xff; + assert_eq!( + children_digest([a.as_slice()]), + children_digest([body_flipped.as_slice()]) + ); + let mut plain = Sha256::new(); + plain.update(b"k"); + let empty: ChildrenDigest = plain.finalize().into(); + assert_eq!(no_children(), empty, "SHA-256 of the domain alone"); + } + + #[test] + fn children_are_listed_in_ordinal_order() { + // 0 is the root; 1 and 3 are its children at ordinals 1 and 0; 2 is + // 1's child; 4 chains from nothing. + let children = children_of([None, Some((0, 1)), Some((1, 0)), Some((0, 0)), None]); + assert_eq!(children, vec![vec![3, 1], vec![2], vec![], vec![], vec![]]); + assert_eq!( + children_of([Some((7, 0))]), + vec![Vec::::new()], + "a parent out of range" + ); + } +} diff --git a/packages/stack-encrypt/src/json/derive.rs b/packages/stack-encrypt/src/json/derive.rs index 1b4753cd3..46a912b73 100644 --- a/packages/stack-encrypt/src/json/derive.rs +++ b/packages/stack-encrypt/src/json/derive.rs @@ -4,10 +4,11 @@ //! One walk ([`walk`]) serves both the index's term operation and the //! sealing operation, so a query derives exactly the selectors a stored //! document carries. The walk is depth-first, pre-order, siblings in key -//! order for an object and in position order for an array; every entry's -//! ordinal `j` is its place in that order, and the AAD of its ciphertext -//! binds it there. -use std::collections::BTreeMap; +//! order for an object and in position order for an array. Sealing runs the +//! other way, from the leaves up, so that each node's AAD can bind the AEAD +//! tags of its children ([`cascade`](super::cascade)); the AAD also binds the +//! slot `j` the entry is stored at. +use std::collections::{BTreeMap, HashMap}; use std::sync::Arc; use serde_json::{Map, Number, Value}; @@ -19,6 +20,7 @@ use vitaminc_hmac::HmacSha256Prf; use vitaminc_prf::{BlockVisitor, IntoPrfContext, PrfValue}; use vitaminc_protected::{AsProtectedRef, Controlled, Protected}; +use super::cascade::{self, ChildrenDigest}; use super::path::{key_step_len, push_key, JsonPath, Segment, ARRAY_STEP}; use super::plain::{ insert_member, protected_array, protected_object, push_element, read_protected, take_protected, @@ -38,8 +40,10 @@ const VALUE_DOMAIN: &[u8] = b"stack-encrypt/json/value/v1"; /// PAE domain of a chain head's token: the root's, or a top-level field's /// under the sealed root. const TOKEN_DOMAIN: &[u8] = b"stack-encrypt/json/token/v1"; -/// PAE domain of an entry's AAD. -const ENTRY_AAD_DOMAIN: &[u8] = b"stack-encrypt/json/entry/v1"; +/// PAE domain of an entry's AAD. `v2` binds the digest of the node's +/// children's AEAD tags ([`cascade`](super::cascade)); `v1` bound the slot +/// as the node's depth-first position and nothing of its children. +const ENTRY_AAD_DOMAIN: &[u8] = b"stack-encrypt/json/entry/v2"; /// PAE domain of an object member's rank under its parent's token, which /// orders the members' ordinals. Keyed, so a token does not let its holder /// test key names. @@ -790,8 +794,17 @@ pub(crate) fn seal_pending<'a, K: 'static>( } /// An entry's AAD: the domain, the keyset, the column's AAD, the key tag, -/// the entry's selector and its ordinal. -fn entry_aad(keyset_id: Uuid, aad: &Context<'_>, tag: &[u8], s: &Tag, j: u64) -> Context<'static> { +/// the entry's selector, the slot `j` it is stored at, and the digest of its +/// children's AEAD tags ([`cascade::children_digest`]). The last is what +/// makes a node's ciphertext commit to everything beneath it. +fn entry_aad( + keyset_id: Uuid, + aad: &Context<'_>, + tag: &[u8], + s: &Tag, + j: u64, + children: &ChildrenDigest, +) -> Context<'static> { Context::pae(&[ ENTRY_AAD_DOMAIN, keyset_id.as_bytes(), @@ -799,6 +812,7 @@ fn entry_aad(keyset_id: Uuid, aad: &Context<'_>, tag: &[u8], s: &Tag, j: u64) -> tag, s, &j.to_le_bytes(), + children, ]) } @@ -910,19 +924,33 @@ fn seal( let tag = key.tag; let Walked { terms, plain } = walked; let nodes = &terms.nodes; - let mut entries = Vec::with_capacity(nodes.len()); - for (j, (node, plain)) in nodes.iter().zip(plain).enumerate() { - // The root's plaintext, the first, is the whole document when the - // root is sealed (`whole` is taken there, and is `None` after); the - // node encoding it replaces is wiped as it drops. - let plaintext = whole.take().unwrap_or(plain); + // The children a node's AAD commits to: the nodes that chain from it, by + // ordinal. Under the sealed root its members chain from nothing, so the + // root commits to no children and opens under the empty digest. + let children = cascade::children_of(nodes.iter().map(|n| n.parent.zip(n.ordinal))); + let mut sealed: Vec> = vec![Vec::new(); nodes.len()]; + // From the leaves up: the walk is pre-order, so every descendant of a + // node has a larger index, and the reverse order seals a node's children + // before the node. + for (i, plain) in plain.into_iter().enumerate().rev() { + // The root's plaintext is the whole document when the root is + // sealed; the node encoding it replaces is wiped as it drops. + let plaintext = if i == 0 { + whole.take().unwrap_or(plain) + } else { + plain + }; + let digest = cascade::children_digest(children[i].iter().map(|&c| sealed[c].as_slice())); let c = cipher .seal( plaintext, - &entry_aad(keyset_id, aad, &tag, &node.s, j as u64), + &entry_aad(keyset_id, aad, &tag, &nodes[i].s, i as u64, &digest), ) .map_err(|_| Error::Aead)?; - let c = c.as_ref().to_vec(); + sealed[i] = c.as_ref().to_vec(); + } + let mut entries = Vec::with_capacity(nodes.len()); + for (node, c) in nodes.iter().zip(sealed) { let p = node .parent .zip(node.ordinal) @@ -1049,7 +1077,8 @@ fn decode_node(plaintext: &Protected>) -> Result { }) } -/// Open one entry. The plaintext the cipher returns stays in its +/// Open one entry stored at slot `j`, under the digest of the children it +/// must have been sealed over. The plaintext the cipher returns stays in its /// `Protected` and is wiped when decoding is done. fn open_entry( cipher: &Aes256Cipher, @@ -1057,11 +1086,12 @@ fn open_entry( aad: &Context<'_>, entry: &JsonEntry, j: u64, + children: &ChildrenDigest, ) -> Result { let plaintext = cipher .open( LocalCipherText::from(entry.c.clone()), - &entry_aad(header.keyset_id, aad, &header.tag, &entry.s, j), + &entry_aad(header.keyset_id, aad, &header.tag, &entry.s, j, children), ) .map_err(|_| Error::Aead)?; decode_node(&plaintext) @@ -1072,61 +1102,6 @@ fn take_label(mut node: Opened) -> (Label, Opened) { (std::mem::replace(&mut node.label, Label::Root), node) } -/// Rebuild the value of `node`, taking its descendants from `rest`, the -/// pre-order sequence after it. Every child is taken with `next`, so each -/// step consumes an entry and the rebuild ends on any sequence. The value -/// is built under `Protected`: each member and element is moved in from -/// its own, so a rebuild that fails part-way wipes what it had. -/// -/// `depth` is `node`'s depth below the rebuild's start. Each entry is -/// authentic, but which entries the server returned, and so how deep the -/// containers it lines up nest, is not: rows that repeat one container, or -/// list only the containers of a wide document, would otherwise recurse as -/// deep as there are rows. Nothing the index seals nests deeper than -/// [`MAX_DEPTH`], so deeper is refused as a shape mismatch. -fn rebuild( - node: Opened, - rest: &mut impl Iterator, - depth: usize, -) -> Result, Error> { - if let Some(whole) = node.whole { - return Ok(whole); - } - if depth > MAX_DEPTH { - return Err(shape("a JSON document no deeper than the index seals")); - } - match node.kind { - Kind::Object => { - let mut object = protected_object(); - for _ in 0..node.children { - let (label, member) = take_label(next_child(rest)?); - let Label::Key(key) = label else { - return Err(shape("an object member")); - }; - let value = rebuild(member, rest, depth + 1)?; - insert_member(&mut object, key, value); - } - Ok(object) - } - Kind::Array => { - let mut array = protected_array(node.children as usize); - for i in 0..node.children { - let element = next_child(rest)?; - if !matches!(element.label, Label::Index(at) if at == u64::from(i)) { - return Err(shape("an array element in order")); - } - push_element(&mut array, rebuild(element, rest, depth + 1)?); - } - Ok(array) - } - _ => node.scalar.ok_or_else(|| shape("a JSON scalar")), - } -} - -fn next_child(rest: &mut impl Iterator) -> Result { - rest.next().ok_or_else(|| shape("a complete JSON subtree")) -} - /// The value the caller asked for, out of the `Protected` that held it: /// the one place an opened JSON plaintext is unwrapped. fn hand_over(value: Protected) -> Value { @@ -1139,7 +1114,14 @@ fn retrieval( header: &JsonHeader, context: AeadContext, ) -> Result<(Request, Context<'static>), Error> { - let piece = context.into_context(); + retrieval_of(header, context.into_context()) +} + +/// [`retrieval`] over a context already split into its piece. +fn retrieval_of( + header: &JsonHeader, + piece: vitaminc_aead::ContextPiece<'static>, +) -> Result<(Request, Context<'static>), Error> { let descriptor = Descriptor::from_piece(&piece); descriptor.check()?; Ok(( @@ -1148,46 +1130,120 @@ fn retrieval( )) } -/// Open the document: one key retrieval, then the sealed root alone, or -/// every entry rebuilt into the value. +/// Open the document: one key retrieval, then the sealed root alone, or, +/// with the root open, every entry placed by the chain and opened under the +/// cascade. +/// +/// With the root open the client walks the chain itself, as the server does +/// for a read of `$`: it derives the root's token, finds each node's +/// children by their parent tags, and opens each node under the digest of +/// its children's tags ([`place`]). Deriving the root's token takes the +/// index key of the keyset the document was sealed under. fn open_document<'a, K>( doc: JsonDocument, cipher: &'a StackCipher, context: AeadContext, ) -> Pending<'a, Value, K> { - let (request, aad) = match retrieval(&doc.header, context) { + let piece = context.into_context(); + let prf_context = piece.clone().into_prf_context().into_owned(); + let (request, aad) = match retrieval_of(&doc.header, piece) { Ok(parts) => parts, Err(error) => return Pending::failed(cipher, error), }; + let keyset = if doc.header.sealed_root { + None + } else { + match cipher.loaded_keyset(doc.header.keyset_id) { + Some(state) => Some(state), + None => { + return Pending::failed( + cipher, + Error::Other( + "opening a JSON document with its root open needs its keyset loaded".into(), + ), + ) + } + } + }; Pending::request(cipher, vec![request], move |responses| { let key = responses.next_retrieved_key()?; let aes = leaf_cipher(&key)?; - if doc.header.sealed_root { - let root = doc - .entries - .first() - .ok_or_else(|| shape("a document with a root entry"))?; - return open_entry(&aes, &doc.header, &aad, root, 0)? - .whole - .map(hand_over) - .ok_or_else(|| shape("a sealed root")); + match keyset { + None => open_sealed_root(&doc, &aes, &aad), + Some(keyset) => open_whole_chain(&doc, &aes, &aad, Selectors::new(keyset, prf_context)), } - let opened = doc + }) +} + +/// The whole document out of its sealed root, which is stored at slot `0` +/// and commits to no children. +fn open_sealed_root( + doc: &JsonDocument, + aes: &Aes256Cipher, + aad: &Context<'_>, +) -> Result { + let root = doc + .entries + .first() + .ok_or_else(|| shape("a document with a root entry"))?; + open_entry(aes, &doc.header, aad, root, 0, &cascade::no_children())? + .whole + .map(hand_over) + .ok_or_else(|| shape("a sealed root")) +} + +/// The whole document with its root open: every entry placed by the chain +/// from the root's token, as one read of `$` that must reach every entry. +fn open_whole_chain( + doc: &JsonDocument, + aes: &Aes256Cipher, + aad: &Context<'_>, + selectors: Selectors, +) -> Result { + let root = JsonPath::root(); + let token = prf16( + &selectors.keyset.prf, + TOKEN_DOMAIN, + &selectors.context, + &root.token_bytes().unwrap_or_default(), + )?; + // A document whose chain the root's token does not enter was sealed + // under another column or keyset, or its tags were rewritten. Opening + // its first entry lets the cipher say which, as it did before the chain + // was walked, rather than reporting rows no read reached. + if doc.find(&token).is_none() { + let first = doc .entries - .iter() - .enumerate() - .map(|(j, entry)| open_entry(&aes, &doc.header, &aad, entry, j as u64)) - .collect::, _>>()?; - let mut rest = opened.into_iter(); - let root = rest - .next() + .first() .ok_or_else(|| shape("a document with a root entry"))?; - let value = rebuild(root, &mut rest, 0)?; - if rest.next().is_some() { - return Err(shape("a document with no entries past its root")); - } - Ok(hand_over(value)) - }) + let _ = open_entry(aes, &doc.header, aad, first, 0, &cascade::no_children())?; + return Err(shape("a document whose chain reaches its root")); + } + let rows = doc + .entries + .iter() + .enumerate() + .map(|(j, entry)| Row { + entry: entry.clone(), + j: j as u64, + d: None, + }) + .collect(); + let reads = [Read { + token, + path: root, + probe: false, + }]; + let opening = Opening { + header: &doc.header, + aad, + aes, + }; + let mut values = place(rows, &reads, &selectors, 0, &opening)?; + match (values.pop(), values.is_empty()) { + (Some(value), true) => Ok(hand_over(value)), + _ => Err(shape("a document whose chain reaches its root")), + } } impl DecryptInto for JsonDocument { @@ -1208,8 +1264,15 @@ fn wrong(expected: &'static str) -> Error { /// [`Error::JsonWrongEntry`] before any key is retrieved: it is another /// path's entry, or the sealed root (whose selector is the root's) in place /// of a node, and it would open, since every entry is sealed under its own -/// selector. A container node holds no value of its own and is refused -/// with [`Error::OpenedType`]; its subtree opens through [`JsonSubtree`]. +/// selector. +/// +/// The entry opens under the digest of no children, which is what a +/// scalar, an empty container and the sealed root are sealed under. A +/// container with children is sealed over its children's tags, which `->` +/// does not return, so it does not open alone: it is refused by the cipher +/// with [`Error::Aead`], and its value opens through [`JsonSubtree`]. An +/// empty container opens and holds no value of its own: +/// [`Error::OpenedType`]. impl DecryptInto for JsonExtracted { type Context = AeadContext; fn decryption(self, context: Self::Context) -> Decryption { @@ -1225,7 +1288,14 @@ impl DecryptInto for JsonExtracted { Pending::request(cipher, vec![request], move |responses| { let key = responses.next_retrieved_key()?; let aes = leaf_cipher(&key)?; - let opened = open_entry(&aes, &node.header, &aad, &node.entry, node.j)?; + let opened = open_entry( + &aes, + &node.header, + &aad, + &node.entry, + node.j, + &cascade::no_children(), + )?; opened .whole .or(opened.scalar) @@ -1320,15 +1390,12 @@ impl DecryptInto for JsonSubtree { } } -/// One row of a read, opened: the node, and the selector, self tag, -/// position and depth the server returned it with. The depth is the -/// server's report and optional on the wire; the rows are placed by their -/// positions and the opened nodes' child counts, and a depth that disagrees -/// with the place a row is put in is refused. +/// One row of a read as the server returned it, not yet opened: the entry, +/// the slot `j` its ciphertext is bound to, and the depth the server +/// reported, which is optional on the wire and checked against the place +/// the chain puts the row in. struct Row { - node: Opened, - s: Tag, - x: super::SelfTag, + entry: JsonEntry, j: u64, d: Option, } @@ -1366,107 +1433,273 @@ fn open_subtree<'a, K>( let key_bytes = responses.next_retrieved_key()?; let aes = leaf_cipher(&key_bytes)?; let rows = nodes - .iter() - .map(|n| { - Ok(Row { - node: open_entry(&aes, &header, &aad, &n.entry, n.j)?, - s: n.entry.s, - x: n.entry.x, - j: n.j, - d: n.depth, - }) + .into_iter() + .map(|n| Row { + entry: n.entry, + j: n.j, + d: n.depth, }) - .collect::, Error>>()?; - place(rows, &key, &header.iv) + .collect(); + let opening = Opening { + header: &header, + aad: &aad, + aes: &aes, + }; + place(rows, &key.reads, &key.selectors, key.bound, &opening) }) } -/// What [`rebuild_walked`] checks a row's derived fields with: the selectors -/// of the key's column, and the document's `iv`, which salts every self tag. -struct Check<'k> { - selectors: &'k Selectors, - iv: &'k [u8], +/// The cipher side of opening a document's rows: its header, the AAD its +/// entries open under, and the cipher of its data key. +struct Opening<'o> { + header: &'o JsonHeader, + aad: &'o Context<'o>, + aes: &'o Aes256Cipher, +} + +/// Where a node sits, for the check of its selector: at the top of a read, +/// named by a concrete path, or beneath a node whose selector bytes are +/// known. +enum Under<'p> { + Path(&'p JsonPath), + Node(&'p Protected>), } -/// Put the opened rows of a read back into the nodes its tokens named, in -/// the key's order, refusing any row that is not where the key says the -/// read's rows are ([`Error::JsonWrongEntry`]), and the whole read with -/// [`Error::JsonWildcardTruncated`] if a probe named a node. +/// Put the rows of a read back into the nodes its `reads` named, in the +/// reads' order, opening every node under the cascade +/// ([`cascade`](super::cascade)); and refuse the whole read with +/// [`Error::JsonWildcardTruncated`] at `bound` if a probe named a node. /// -/// Each entry opened under its own selector `s` and position `j`, which its -/// ciphertext is bound to, so each is a genuine node of the row; what the -/// check adds is that it is the node the read asked for. A read's rows are, -/// for each token that names a node, that node at depth `0` and then every -/// node beneath it at consecutive positions, each node's children in order, -/// and the key's tokens are in document order. So the rows are placed one -/// by one. A node at depth `0` must carry the self tag of a token of the -/// key not yet matched, and that token's path's selector and last step (its -/// key, its position, or the root). Every row beneath it must be the next -/// position after the one before, one level below its parent, carry the -/// selector of its parent's path extended by its own label, and carry the -/// self tag of the token its label derives under its parent's. +/// The rows arrive in any order, and nothing here reads a row's slot `j` as +/// a position in the document: the chain's tags say where each row sits. The +/// node a read's token names is the row carrying that token's self tag, and +/// a node's children are the rows carrying its parent tags at the ordinals +/// `0, 1, 2, …`, each with the token the member or element rule derives for +/// it, as the server finds them. The tags are the database's word, so they +/// only say where to try each row. What decides is the cipher: every node +/// opens under the digest of the AEAD tags of the children the tags gave +/// it, so a node whose children were swapped, dropped, added or reordered, +/// or which was put under the wrong parent, does not open +/// ([`Error::Aead`]). /// -/// The self tags are not authenticated, and the database holds every token -/// the read sent and every row's `iv`, so matching them is a consistency -/// check, which catches a corrupted row or a server that walked wrongly, -/// not a dishonest database. The selector, the position and the label, -/// which the AAD and the ciphertext cover, are what bind a row to its -/// place. Still allowed by design: an entry from another row of the -/// column, returned with that row's header (ADR 0010), and a node at the -/// same selector under another element (`$.items[0].price` for -/// `$.items[1].price`), which no bound field tells apart. +/// The checks the client still makes, each [`Error::JsonWrongEntry`]: no +/// row is placed twice; every row the server returned is one the reads +/// reach; the depth the server reported for a row is the one the chain +/// gives it; a node's selector is its path's, and its decrypted label is the +/// last step of that path; a child's self tag is the one its label's rule +/// derives; the sealed root sits nowhere but at `$`; and a container has the +/// number of children its plaintext says. Under the cascade most of these +/// hold whenever every node opens; they are kept because the node at the +/// top of a read is checked by nothing above it, and because each is +/// cheaper than the AEAD it backs. +/// +/// Still allowed by design (ADR 0010): an entry from another row of the +/// column, returned with that row's header; a node at the same selector +/// under another element (`$.items[0].price` for `$.items[1].price`) at the +/// top of a read, which no bound field tells apart; and rows withheld whole. fn place( rows: Vec, - key: &JsonSubtreeKey, - iv: &[u8], + reads: &[Read], + selectors: &Selectors, + bound: u32, + opening: &Opening<'_>, ) -> Result>, Error> { - let check = Check { - selectors: &key.selectors, - iv, - }; - let refused = || wrong("a node the walked path reaches"); - let mut rest = rows.into_iter(); - // The first token of the key not yet matched to a row. - let mut next = 0; + let mut placing = Placing::new(rows, selectors, opening); let mut reached = Vec::new(); let mut truncated = false; - while let Some(top) = rest.next() { - if top.d.is_some_and(|d| d != 0) { - return Err(refused()); - } - let at = key.reads[next..] - .iter() - .position(|read| self_tag(iv, &read.token) == top.x) - .ok_or_else(refused)? - + next; - next = at + 1; - let read = &key.reads[at]; - let bytes = Protected::new(read.path.selector_bytes()); - if key.selectors.of(bytes.clone())? != top.s || !names(&read.path, &top.node.label) { - return Err(refused()); - } - let mut cursor = top.j.checked_add(1).ok_or_else(refused)?; - let value = rebuild_walked( - top.node, - &bytes, - &read.token, - 0, - &mut rest, - &mut cursor, - &check, - )?; + for read in reads { + let Some(at) = placing.find(&read.token) else { + continue; + }; + let (value, _) = placing.node(at, &read.token, Under::Path(&read.path), 0)?; if read.probe { truncated = true; } else { reached.push(value); } } + // A read returns the subtrees of the nodes its tokens name and nothing + // else, so a row none of them reaches is not an answer to it. + if placing.used.contains(&false) { + return Err(wrong("rows the walked paths reach")); + } if truncated { - return Err(Error::JsonWildcardTruncated { bound: key.bound }); + return Err(Error::JsonWildcardTruncated { bound }); } Ok(reached) } +/// The rows of one read being placed: which have been placed, and the +/// server's two lookups over them, by self tag and by parent tag. +struct Placing<'k> { + rows: Vec, + used: Vec, + by_x: HashMap, + by_p: HashMap, + selectors: &'k Selectors, + opening: &'k Opening<'k>, +} + +impl<'k> Placing<'k> { + fn new(rows: Vec, selectors: &'k Selectors, opening: &'k Opening<'k>) -> Self { + // Where several rows carry one tag, which no sealing produces, the + // last of them, as the server's index keeps it + // (`eql_v4_internal.json_tree_index` builds it with + // `jsonb_object_agg`), so the client tries the row the server found. + let mut by_x = HashMap::with_capacity(rows.len()); + let mut by_p = HashMap::with_capacity(rows.len()); + for (i, row) in rows.iter().enumerate() { + let _ = by_x.insert(row.entry.x, i); + if let Some(p) = row.entry.p { + let _ = by_p.insert(p, i); + } + } + Self { + used: vec![false; rows.len()], + rows, + by_x, + by_p, + selectors, + opening, + } + } + + /// The row carrying the self tag of `token`. + fn find(&self, token: &Tag) -> Option { + self.by_x + .get(&self_tag(&self.opening.header.iv, token)) + .copied() + } + + /// The children of the node whose token is `token`, as the server + /// enumerates them: the rows at its parent tags for the ordinals from + /// zero until the first miss, each with the token its self tag says it + /// has. A child whose self tag fits neither rule is refused: the server + /// would not descend into it, and no sealing writes one. + fn children(&self, token: &Tag) -> Result, Error> { + let iv = &self.opening.header.iv; + let mut children = Vec::new(); + // A node has fewer children than there are rows, so the probe loop + // is bounded whatever the tags say. + for o in 0..u32::try_from(self.rows.len()).unwrap_or(u32::MAX) { + let Some(&child) = self.by_p.get(&parent_tag(iv, token, o)) else { + break; + }; + let child_token = hit_token(iv, token, &self.rows[child].entry, o) + .ok_or_else(|| wrong("a child the member or element rule names"))?; + children.push((child, child_token)); + } + Ok(children) + } + + /// Place and open the row at `at`, the node whose token is `token`, + /// sitting `under` its parent `d` levels below the node its read named: + /// its children found by the chain, then the node opened under the + /// digest of their tags, then the checks, then the children in turn. + /// Returns the node's value and its label, which its parent places it + /// by. + /// + /// The recursion follows only nodes that opened, so it follows the tree + /// that was sealed, which nests no deeper than [`MAX_DEPTH`]. + fn node( + &mut self, + at: usize, + token: &Tag, + under: Under<'_>, + d: u32, + ) -> Result<(Protected, Label), Error> { + let refused = || wrong("a node the walked path reaches"); + // An honest walk reaches each row once. + if std::mem::replace(&mut self.used[at], true) { + return Err(wrong("rows of distinct entries")); + } + if self.rows[at].d.is_some_and(|got| got != d) { + return Err(wrong("a row at the depth the chain gives it")); + } + let children = self.children(token)?; + let digest = cascade::children_digest( + children + .iter() + .map(|&(child, _)| self.rows[child].entry.c.as_slice()), + ); + let row = &self.rows[at]; + let Opening { header, aad, aes } = *self.opening; + let opened = open_entry(aes, header, aad, &row.entry, row.j, &digest)?; + // The sealed root is the whole document, read only at `$`. + if opened.whole.is_some() + && !matches!(under, Under::Path(path) if path.segments().is_empty()) + { + return Err(wrong("a node of the walked subtree")); + } + if opened.children as usize != children.len() { + return Err(wrong("a container with the children it was sealed over")); + } + let (label, opened) = take_label(opened); + // The selector the chain gives this node: the read's path for the + // node it named, its parent's path extended by its own label below. + let bytes = match under { + Under::Path(path) => { + if !names(path, &label) { + return Err(refused()); + } + Protected::new(path.selector_bytes()) + } + Under::Node(parent) => child_selector_bytes(parent, &label).ok_or_else(refused)?, + }; + if self.selectors.of(bytes.clone())? != row.entry.s { + return Err(refused()); + } + if let Some(whole) = opened.whole { + return Ok((whole, label)); + } + match opened.kind { + Kind::Object => { + let mut object = protected_object(); + for (child, child_token) in children { + let (value, child_label) = self.child(child, token, &child_token, &bytes, d)?; + let Label::Key(key) = child_label else { + return Err(shape("an object member")); + }; + insert_member(&mut object, key, value); + } + Ok((object, label)) + } + Kind::Array => { + let mut array = protected_array(children.len()); + for (i, (child, child_token)) in children.into_iter().enumerate() { + let (value, child_label) = self.child(child, token, &child_token, &bytes, d)?; + if !matches!(child_label, Label::Index(at) if at == i as u64) { + return Err(shape("an array element in order")); + } + push_element(&mut array, value); + } + Ok((array, label)) + } + _ => Ok((opened.scalar.ok_or_else(|| shape("a JSON scalar"))?, label)), + } + } + + /// Place and open a child, and check that the token its self tag chose + /// (the member or the element rule) is the one its decrypted label + /// derives under its parent's: a member's by its key, an element's by + /// its index. + fn child( + &mut self, + child: usize, + parent_token: &Tag, + child_token: &Tag, + parent_bytes: &Protected>, + d: u32, + ) -> Result<(Protected, Label), Error> { + let (value, label) = self.node(child, child_token, Under::Node(parent_bytes), d + 1)?; + let s = self.rows[child].entry.s; + if labelled_token(parent_token, &s, &label).as_ref() != Some(child_token) { + return Err(wrong("a child under the rule its label names")); + } + Ok((value, label)) + } +} + /// Whether a node with `label` is the one the concrete `path` names by its /// last step: the key, the position, or the root for `$`. fn names(path: &JsonPath, label: &Label) -> bool { @@ -1516,104 +1749,6 @@ fn labelled_token(parent: &Tag, s: &Tag, label: &Label) -> Option { } } -/// [`rebuild`] for a read's rows, checking each child as it is taken: that -/// it is at `cursor`, the position after the row before it; that it is one -/// level below `node`, which sits `d` levels below the node a token named; -/// that it carries the selector of `node`'s path, whose selector bytes are -/// `bytes`, extended by its own label; and that it carries the self tag of -/// the token its label derives under `token`, `node`'s. A sealed root -/// among the rows is not a node of any walk. Rows that end before `node`'s -/// authenticated child count is met are refused like a gap: a read returns -/// a node's whole subtree, so the database left the next node out. -/// -/// The rows are consecutive entries of one sealed document, placed by its -/// authenticated child counts, so they nest no deeper than the document -/// does, which is never deeper than [`MAX_DEPTH`]; a document that does is -/// refused as a shape mismatch rather than recursed into. -fn rebuild_walked( - node: Opened, - bytes: &Protected>, - token: &Tag, - d: u32, - rest: &mut impl Iterator, - cursor: &mut u64, - check: &Check<'_>, -) -> Result, Error> { - if node.whole.is_some() { - return Err(wrong("a node of the walked subtree")); - } - if d as usize > MAX_DEPTH { - return Err(shape("a JSON document no deeper than the index seals")); - } - let next = || wrong("the next node of the walked subtree"); - let child = |rest: &mut dyn Iterator, cursor: &mut u64| { - // A read returns a node's whole subtree and stops at no budget - // short of it (the SQL falls back to the sealed root instead), so - // rows that run out before a child count is met are a row left out - // at the end, the same refusal as one left out in the middle. - let row = rest.next().ok_or_else(next)?; - let child_bytes = child_selector_bytes(bytes, &row.node.label).ok_or_else(next)?; - if row.j != *cursor - || row.d.is_some_and(|got| Some(got) != d.checked_add(1)) - || check.selectors.of(child_bytes.clone())? != row.s - { - return Err(next()); - } - let child_token = labelled_token(token, &row.s, &row.node.label).ok_or_else(next)?; - if self_tag(check.iv, &child_token) != row.x { - return Err(next()); - } - *cursor = cursor.checked_add(1).ok_or_else(next)?; - Ok::<_, Error>((row.node, child_bytes, child_token)) - }; - match node.kind { - Kind::Object => { - let mut object = protected_object(); - for _ in 0..node.children { - let (member, member_bytes, member_token) = child(rest, cursor)?; - let (label, member) = take_label(member); - let Label::Key(key) = label else { - return Err(shape("an object member")); - }; - let value = rebuild_walked( - member, - &member_bytes, - &member_token, - d + 1, - rest, - cursor, - check, - )?; - insert_member(&mut object, key, value); - } - Ok(object) - } - Kind::Array => { - let mut array = protected_array(node.children as usize); - for i in 0..node.children { - let (element, element_bytes, element_token) = child(rest, cursor)?; - if !matches!(element.label, Label::Index(at) if at == u64::from(i)) { - return Err(shape("an array element in order")); - } - push_element( - &mut array, - rebuild_walked( - element, - &element_bytes, - &element_token, - d + 1, - rest, - cursor, - check, - )?, - ); - } - Ok(array) - } - _ => node.scalar.ok_or_else(|| shape("a JSON scalar")), - } -} - /// The key's reads evaluated over the whole document a read fell back to, /// to what the walked rows open to: each concrete path's node, moved out of /// the document's `Protected` into one of its own (leaving `null` behind, @@ -1669,7 +1804,14 @@ fn open_fallback<'a, K>( Pending::request(cipher, vec![request], move |responses| { let retrieved = responses.next_retrieved_key()?; let aes = leaf_cipher(&retrieved)?; - let opened = open_entry(&aes, &root.header, &aad, &root.entry, root.j)?; + let opened = open_entry( + &aes, + &root.header, + &aad, + &root.entry, + root.j, + &cascade::no_children(), + )?; let whole = opened.whole.ok_or_else(|| shape("a sealed root"))?; Ok((key, whole)) }) diff --git a/packages/stack-encrypt/src/json/mod.rs b/packages/stack-encrypt/src/json/mod.rs index a145b913d..c19a5c46e 100644 --- a/packages/stack-encrypt/src/json/mod.rs +++ b/packages/stack-encrypt/src/json/mod.rs @@ -118,6 +118,7 @@ use vitaminc_protected::OpaqueDebug; use crate::target::{DecryptField, DecryptInto, Decryptable, Decryption}; +mod cascade; mod derive; mod path; mod plain; From d2579633cf32a737a861f71028a6b4927bd61882 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 12:01:03 +1100 Subject: [PATCH 03/22] feat(stack-encrypt): store a JSON document's entries in a random order MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A json_tree row stored its entries depth-first, root first, members in key byte order. Every entry carries its path selector, so the stored order alone gave anyone holding the table each row's tree: the parent of every node, the size of every subtree, and which of two sibling key names sorts first (#1357). The containment projection q restated it. With the cascade (the previous commit) nothing reads a slot as a position any more: the server finds entries by tag and reads q as sets, and the client places a read's rows by the chain and verifies them by the tags. So the entries are now stored in a uniformly random order per document, with the root kept at slot 0 where the sealed-root fallback looks for it, and q's pair lists and element groups are shuffled too. The AAD still binds each entry's slot. This is the format, not an option: no header flag, no data-plan option, no switch. The permutation is drawn at sealing time and thrown away; nothing derives it, keyed or not, because nothing needs it and anything it were derived from would have to be kept from the database. Production draws from vitaminc's SafeRand::from_entropy, one generator per document, Fisher–Yates over slots 1..n with next_below (Lemire's reduction, bias at most n / 2^64). The generator is not part of the format: nothing stored depends on it, so it can be replaced without a version change. For fixtures, `StackCipherBuilder::json_order_seed` (behind the `test-support` feature) draws every document's order from one generator started at a fixed seed (SafeRand::from_controlled_seed), and the test helper that builds a cipher over the deterministic key source sets it, so a pinned vector repeats. The one SQL meaning that moves: `->`, `->>` and ordering by `eql_v4.ord_term(col -> sel)` at a wildcard path return the first entry in the stored order carrying the selector, now an unspecified element of the row (the same one every time) rather than element 0. Every element is reached through all its matches; the rustdoc of the extraction says so. Tests that pick an entry by its depth-first slot are rewritten in a later commit. Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- Cargo.lock | 1 + languages/golang/encrypt/guest/Cargo.lock | 1 + packages/eql/Cargo.lock | 1 + packages/stack-encrypt/Cargo.toml | 2 + packages/stack-encrypt/fuzz/Cargo.lock | 1 + packages/stack-encrypt/src/cipher.rs | 32 ++++ packages/stack-encrypt/src/json/derive.rs | 39 +++- packages/stack-encrypt/src/json/mod.rs | 14 +- packages/stack-encrypt/src/json/order.rs | 166 ++++++++++++++++++ packages/stack-encrypt/src/json/query.rs | 13 ++ packages/stack-encrypt/tests/common/mod.rs | 14 +- .../json_properties.proptest-regressions | 7 + 12 files changed, 275 insertions(+), 16 deletions(-) create mode 100644 packages/stack-encrypt/src/json/order.rs create mode 100644 packages/stack-encrypt/tests/json_properties.proptest-regressions diff --git a/Cargo.lock b/Cargo.lock index 271420eac..a77b94a88 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3419,6 +3419,7 @@ dependencies = [ "tokio", "trybuild", "uuid", + "vitaminc 0.8.0", "vitaminc-aead 0.8.0", "vitaminc-aead-frame", "vitaminc-encrypt 0.8.0", diff --git a/languages/golang/encrypt/guest/Cargo.lock b/languages/golang/encrypt/guest/Cargo.lock index 34e8e7007..fa249ed7d 100644 --- a/languages/golang/encrypt/guest/Cargo.lock +++ b/languages/golang/encrypt/guest/Cargo.lock @@ -2174,6 +2174,7 @@ dependencies = [ "stack-kms", "thiserror 1.0.69", "uuid", + "vitaminc 0.8.0", "vitaminc-aead 0.8.0", "vitaminc-aead-frame", "vitaminc-encrypt 0.8.0", diff --git a/packages/eql/Cargo.lock b/packages/eql/Cargo.lock index 06e657798..ba95bebae 100644 --- a/packages/eql/Cargo.lock +++ b/packages/eql/Cargo.lock @@ -4279,6 +4279,7 @@ dependencies = [ "stack-kms", "thiserror 1.0.69", "uuid", + "vitaminc 0.8.0", "vitaminc-aead 0.8.0", "vitaminc-aead-frame", "vitaminc-encrypt 0.8.0", diff --git a/packages/stack-encrypt/Cargo.toml b/packages/stack-encrypt/Cargo.toml index 7ed76d784..598cdaa5b 100644 --- a/packages/stack-encrypt/Cargo.toml +++ b/packages/stack-encrypt/Cargo.toml @@ -42,6 +42,8 @@ vitaminc-normalize = { workspace = true } vitaminc-ore = { workspace = true } vitaminc-prf = { workspace = true } vitaminc-protected = { workspace = true } +# The generator that draws a JSON document's stored order (`json::order`). +vitaminc = { workspace = true } # `dynamic` only: serde over the transport frame, which reads a binding's # control data (statistics, options, a plan) into typed messages. vitaminc-aead-frame = { workspace = true, optional = true, features = ["serde"] } diff --git a/packages/stack-encrypt/fuzz/Cargo.lock b/packages/stack-encrypt/fuzz/Cargo.lock index a8647f932..e9642b34c 100644 --- a/packages/stack-encrypt/fuzz/Cargo.lock +++ b/packages/stack-encrypt/fuzz/Cargo.lock @@ -2090,6 +2090,7 @@ dependencies = [ "stack-kms", "thiserror 1.0.69", "uuid", + "vitaminc 0.8.0", "vitaminc-aead 0.8.0", "vitaminc-aead-frame", "vitaminc-encrypt 0.8.0", diff --git a/packages/stack-encrypt/src/cipher.rs b/packages/stack-encrypt/src/cipher.rs index 4609ad412..1515f6463 100644 --- a/packages/stack-encrypt/src/cipher.rs +++ b/packages/stack-encrypt/src/cipher.rs @@ -531,6 +531,9 @@ pub struct StackCipher { /// Every other keyset this cipher has selected, least recently used /// first out. See [`keyset`](Self::keyset). keysets: Mutex, + /// Where the stored orders of the JSON documents this cipher seals are + /// drawn from: fresh entropy per document, outside tests. + json_order: crate::json::order::OrderSource, } #[cfg(feature = "http")] @@ -602,6 +605,11 @@ impl StackCipher { } } + /// Where this cipher draws a JSON document's stored order from. + pub(crate) fn json_order(&self) -> &crate::json::order::OrderSource { + &self.json_order + } + fn keysets(&self) -> std::sync::MutexGuard<'_, KeysetCache> { // The cache holds no invariant a panic mid-update could break (an // insert is two map writes, and a stale name entry only points at @@ -780,6 +788,7 @@ pub struct StackCipherBuilder { kms: K, cache_size: NonZeroUsize, name_ttl: Duration, + json_order_seed: Option>, } impl StackCipherBuilder { @@ -793,6 +802,7 @@ impl StackCipherBuilder { kms: FromEnv, cache_size: KeysetCache::DEFAULT_CAPACITY, name_ttl: DEFAULT_NAME_TTL, + json_order_seed: None, } } } @@ -827,6 +837,22 @@ impl StackCipherBuilder { self.name_ttl = ttl; self } + + /// Test support: draw the stored order of every JSON document the + /// cipher seals from one generator started at `seed`, so that a run + /// sealing the same documents in the same sequence stores them in the + /// same orders, and a fixture can pin them. What the deterministic key + /// source (`kms::DeterministicSource`) is paired with. + /// + /// Never for data: a document's stored order is drawn from the + /// operating system's entropy precisely so that it says nothing, and an + /// order that follows from a known seed hides nothing of the document's + /// shape (see `json::order`). + #[cfg(feature = "test-support")] + pub fn json_order_seed(mut self, seed: [u8; 32]) -> Self { + self.json_order_seed = Some(Protected::new(seed)); + self + } } impl StackCipherBuilder { @@ -842,6 +868,7 @@ impl StackCipherBuilder { kms, cache_size: self.cache_size, name_ttl: self.name_ttl, + json_order_seed: self.json_order_seed, } } @@ -884,6 +911,11 @@ impl StackCipherBuilder { Arc::clone(&default), )), default, + json_order: self + .json_order_seed + .map_or(crate::json::order::OrderSource::Entropy, |seed| { + crate::json::order::OrderSource::controlled(seed) + }), }) } } diff --git a/packages/stack-encrypt/src/json/derive.rs b/packages/stack-encrypt/src/json/derive.rs index 46a912b73..1fc043664 100644 --- a/packages/stack-encrypt/src/json/derive.rs +++ b/packages/stack-encrypt/src/json/derive.rs @@ -14,6 +14,7 @@ use std::sync::Arc; use serde_json::{Map, Number, Value}; use sha2::{Digest, Sha256}; use uuid::Uuid; +use vitaminc::random::SafeRand; use vitaminc_aead::{Cipher, Context, IntoAad, IntoContext, LocalCipherText}; use vitaminc_encrypt::{Aes256Cipher, Key as AesKey}; use vitaminc_hmac::HmacSha256Prf; @@ -21,6 +22,7 @@ use vitaminc_prf::{BlockVisitor, IntoPrfContext, PrfValue}; use vitaminc_protected::{AsProtectedRef, Controlled, Protected}; use super::cascade::{self, ChildrenDigest}; +use super::order; use super::path::{key_step_len, push_key, JsonPath, Segment, ARRAY_STEP}; use super::plain::{ insert_member, protected_array, protected_object, push_element, read_protected, take_protected, @@ -783,12 +785,16 @@ pub(crate) fn seal_pending<'a, K: 'static>( }; let keyset_id = cipher.keyset_id(); let sealed_root = options.sealed_root; + let order = match cipher.cipher().json_order().document() { + Ok(order) => order, + Err(error) => return Pending::failed(cipher, error), + }; Pending::request( cipher, vec![Request::generate_under(descriptor)], move |responses| { let key = responses.next_generated_key()?; - seal(walked, whole, keyset_id, key, &aad, sealed_root) + seal(walked, whole, keyset_id, key, &aad, sealed_root, order) }, ) } @@ -906,6 +912,9 @@ fn sealed_root_plaintext(value: &Value) -> Result>, serde_json }) } +/// Seal the walked nodes under `key`, from the leaves up, each at the slot +/// `order` draws for it ([`order::slots`]), and shuffle the projection with +/// the same generator. fn seal( walked: Walked, mut whole: Option>>, @@ -913,6 +922,7 @@ fn seal( key: stack_kms::DataKeyWithTag, aad: &Context<'_>, sealed_root: bool, + mut order: SafeRand, ) -> Result { if key.tag.len() > usize::from(u16::MAX) { return Err(Error::Other( @@ -928,6 +938,8 @@ fn seal( // ordinal. Under the sealed root its members chain from nothing, so the // root commits to no children and opens under the empty digest. let children = cascade::children_of(nodes.iter().map(|n| n.parent.zip(n.ordinal))); + // Where each node is stored, by its index in the walk; the AAD binds it. + let slots = order::slots(nodes.len(), &mut order); let mut sealed: Vec> = vec![Vec::new(); nodes.len()]; // From the leaves up: the walk is pre-order, so every descendant of a // node has a larger index, and the reverse order seals a node's children @@ -944,18 +956,19 @@ fn seal( let c = cipher .seal( plaintext, - &entry_aad(keyset_id, aad, &tag, &nodes[i].s, i as u64, &digest), + &entry_aad(keyset_id, aad, &tag, &nodes[i].s, slots[i] as u64, &digest), ) .map_err(|_| Error::Aead)?; sealed[i] = c.as_ref().to_vec(); } - let mut entries = Vec::with_capacity(nodes.len()); - for (node, c) in nodes.iter().zip(sealed) { + let mut entries: Vec> = + std::iter::repeat_with(|| None).take(nodes.len()).collect(); + for ((node, c), slot) in nodes.iter().zip(sealed).zip(&slots) { let p = node .parent .zip(node.ordinal) .map(|(parent, o)| parent_tag(&iv, &nodes[parent].token, o)); - entries.push(JsonEntry { + entries[*slot] = Some(JsonEntry { s: node.s, c, x: self_tag(&iv, &node.token), @@ -963,7 +976,10 @@ fn seal( op: node.op.clone(), }); } - let projection = super::query::group_of(nodes, None); + // `slots` is a permutation, so every slot was filled once. + let entries: Vec = entries.into_iter().flatten().collect(); + let mut projection = super::query::group_of(nodes, None); + projection.shuffle(&mut order); Ok(JsonDocument::new( JsonHeader::new(keyset_id, iv, tag, sealed_root), entries, @@ -1273,6 +1289,16 @@ fn wrong(expected: &'static str) -> Error { /// with [`Error::Aead`], and its value opens through [`JsonSubtree`]. An /// empty container opens and holds no value of its own: /// [`Error::OpenedType`]. +/// +/// At a wildcard path (a path with an array step, whose elements all share +/// one selector, at any depth: every element under `$.a[*]`) the entry `->` +/// returns is the first the server finds carrying the selector in the +/// stored order, which is random per document: an unspecified element of +/// the row, the same one every time the row is read, but not element `0`. +/// The same holds for `->>` and for ordering by `eql_v4.ord_term(col -> +/// sel)`. Every element of a wildcard path is reached through all its +/// matches instead (`json_tree_path_query`, or a [`JsonSubtree`] read, +/// which opens them in document order by their labels). impl DecryptInto for JsonExtracted { type Context = AeadContext; fn decryption(self, context: Self::Context) -> Decryption { @@ -2775,6 +2801,7 @@ mod tests { key, &context, true, + SafeRand::from_controlled_seed(Protected::new([0; 32])), ) }; let longest = usize::from(u16::MAX); diff --git a/packages/stack-encrypt/src/json/mod.rs b/packages/stack-encrypt/src/json/mod.rs index c19a5c46e..bf328b1b2 100644 --- a/packages/stack-encrypt/src/json/mod.rs +++ b/packages/stack-encrypt/src/json/mod.rs @@ -120,6 +120,7 @@ use crate::target::{DecryptField, DecryptInto, Decryptable, Decryption}; mod cascade; mod derive; +pub(crate) mod order; mod path; mod plain; mod query; @@ -419,7 +420,8 @@ impl JsonEntry { } /// A stored JSON document: its [`JsonHeader`] and its entries, one per -/// node, in depth-first pre-order with the root first. What +/// node, in a random order drawn for the document when it was sealed, with +/// the root first. What /// [`json_document`](crate::target::json_document) produces and /// [`DecryptInto`](crate::DecryptInto) opens. /// @@ -441,7 +443,10 @@ impl JsonDocument { &self.header } - /// The entries, in depth-first pre-order; the root is first. + /// The entries, in their stored order: the root first, the rest in a + /// uniformly random order drawn when the document was sealed, which says + /// nothing of the tree. A node's children are found by the chain + /// ([`children`](Self::children)), not by position. pub fn entries(&self) -> &[JsonEntry] { &self.entries } @@ -553,8 +558,9 @@ impl JsonDocument { } } - /// The node at ordinal `j` as an extraction returns it: the entry with - /// the header beside it, which opens on its own. + /// The node stored at slot `j` as an extraction returns it: the entry + /// with the header beside it. A scalar or an empty container opens on + /// its own; a container with children opens only with its subtree. pub fn node(&self, j: usize) -> Option { self.entries.get(j).map(|entry| JsonNode { entry: entry.clone(), diff --git a/packages/stack-encrypt/src/json/order.rs b/packages/stack-encrypt/src/json/order.rs new file mode 100644 index 000000000..b72491a9e --- /dev/null +++ b/packages/stack-encrypt/src/json/order.rs @@ -0,0 +1,166 @@ +//! The stored order: where a document's entries sit in `sv`, and the order +//! of its containment projection `q`. +//! +//! A document's entries are stored in a uniformly random order, drawn +//! afresh for each document when it is sealed and then thrown away, with +//! the root at slot `0`, where the sealed-root fallback finds it. Every +//! document has a root, so its slot says nothing. The pairs of each group of +//! `q`, and the element groups of each list, are shuffled the same way. +//! +//! Nothing needs the permutation afterwards. The server finds entries by +//! their self and parent tags and reads `q` as sets (`jsonb @>`), and the +//! client places a read's rows by the chain and verifies them by the cascade +//! ([`cascade`](super::cascade)); the AAD binds each entry's slot, so an +//! entry cannot be moved between slots, but nothing reads a slot as a +//! position in the document. So the permutation is not derived from +//! anything, keyed or not: whatever it were derived from would have to be +//! kept from the database, and nothing is gained by being able to derive +//! it. +//! +//! What the depth-first order used to show and this one does not: each +//! row's tree (the parent of every node), the size of every subtree, and +//! which of two sibling key names sorts first. What any order leaves +//! visible is listed on the leakage page +//! (`packages/eql/docs/reference/json-tree-leakage.md`). +//! +//! The generator is `vitaminc`'s [`SafeRand`] (ChaCha20, seeded from the +//! operating system) and each draw is [`SafeRand::next_below`] (Lemire's +//! reduction: one 64-bit draw per call, bias at most `n / 2⁶⁴`, negligible +//! for any document). Which generator draws the permutation is not part of +//! the format, since nothing stored depends on it, and it can be replaced +//! without a version change. + +use std::sync::Mutex; + +use vitaminc::random::{Rng, SafeRand}; +use vitaminc_protected::{Controlled, Protected}; + +use crate::Error; + +/// Where a cipher draws the stored orders of the documents it seals from. +pub(crate) enum OrderSource { + /// A fresh generator per document, seeded from the operating system: + /// the only source outside tests. + Entropy, + /// A generator per document seeded from one stream that starts at a + /// fixed seed, so a run that seals the same documents in the same order + /// stores them in the same orders. Test support only + /// ([`StackCipherBuilder::json_order_seed`](crate::StackCipherBuilder)). + Controlled(Box>), +} + +impl OrderSource { + /// The source a fixed `seed` starts. + pub(crate) fn controlled(seed: Protected<[u8; 32]>) -> Self { + Self::Controlled(Box::new(Mutex::new(SafeRand::from_controlled_seed(seed)))) + } + + /// The generator one document's order is drawn from. + pub(crate) fn document(&self) -> Result { + match self { + Self::Entropy => SafeRand::from_entropy() + .map_err(|_| Error::Other("no entropy for a JSON document's stored order".into())), + Self::Controlled(stream) => { + // The stream holds no invariant a panic could break. + let mut stream = stream + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let seed = Protected::generate(|| { + let mut seed = [0u8; 32]; + stream.fill_bytes(&mut seed); + seed + }); + Ok(SafeRand::from_controlled_seed(seed)) + } + } + } +} + +/// A uniform shuffle of `items` (Fisher–Yates), each swap index drawn with +/// [`SafeRand::next_below`]. +pub(crate) fn shuffle(items: &mut [T], rng: &mut SafeRand) { + for i in (1..items.len()).rev() { + // `items` is a document's entries, or a group of its projection: + // never more than `u32::MAX` of them, since an ordinal past + // PostgreSQL's integer is refused when the document is walked. + let bound = u32::try_from(i + 1).unwrap_or(u32::MAX); + let j = rng.next_below(bound) as usize; + items.swap(i, j); + } +} + +/// The slot each of `n` nodes is stored at, by its index in the walk: the +/// root (index `0`) at slot `0`, and the other nodes at a uniformly random +/// permutation of the slots `1..n`. +pub(crate) fn slots(n: usize, rng: &mut SafeRand) -> Vec { + let mut slots: Vec = (0..n).collect(); + if let Some(rest) = slots.get_mut(1..) { + shuffle(rest, rng); + } + slots +} + +#[cfg(test)] +mod tests { + use super::*; + + fn seeded(byte: u8) -> SafeRand { + SafeRand::from_controlled_seed(Protected::new([byte; 32])) + } + + #[test] + fn the_slots_are_a_permutation_with_the_root_first() { + for n in [0, 1, 2, 3, 17, 200] { + let slots = slots(n, &mut seeded(1)); + let mut sorted = slots.clone(); + sorted.sort_unstable(); + assert_eq!(sorted, (0..n).collect::>(), "{n}: a permutation"); + if n > 0 { + assert_eq!(slots[0], 0, "{n}: the root at slot 0"); + } + } + assert_ne!( + slots(50, &mut seeded(1)), + (0..50).collect::>(), + "not the identity (with overwhelming probability)" + ); + } + + /// Every arrangement of three items comes up, each about a sixth of the + /// time: the shuffle reaches every permutation and favours none. + #[test] + fn the_shuffle_is_uniform_over_small_permutations() { + let mut rng = seeded(2); + let mut counts = std::collections::BTreeMap::new(); + let runs = 60_000; + for _ in 0..runs { + let mut items = [0, 1, 2]; + shuffle(&mut items, &mut rng); + *counts.entry(items).or_insert(0usize) += 1; + } + assert_eq!(counts.len(), 6, "every permutation of three"); + for (perm, count) in counts { + assert!( + (9_000..11_000).contains(&count), + "{perm:?} came up {count} times in {runs}" + ); + } + } + + #[test] + fn a_controlled_source_repeats_and_entropy_does_not() { + let order = |source: &OrderSource| { + let mut a = source.document().unwrap(); + let mut b = source.document().unwrap(); + (slots(30, &mut a), slots(30, &mut b)) + }; + let one = order(&OrderSource::controlled(Protected::new([3; 32]))); + let two = order(&OrderSource::controlled(Protected::new([3; 32]))); + assert_eq!(one, two, "the same seed, the same orders"); + assert_ne!(one.0, one.1, "each document draws its own order"); + let other = order(&OrderSource::controlled(Protected::new([4; 32]))); + assert_ne!(one, other, "another seed, other orders"); + let fresh = order(&OrderSource::Entropy); + assert_ne!(fresh.0, fresh.1, "entropy draws a new order per document"); + } +} diff --git a/packages/stack-encrypt/src/json/query.rs b/packages/stack-encrypt/src/json/query.rs index c7d9d4075..1b5fd0b3f 100644 --- a/packages/stack-encrypt/src/json/query.rs +++ b/packages/stack-encrypt/src/json/query.rs @@ -42,6 +42,19 @@ pub struct JsonProjection { } impl JsonProjection { + /// Put the pairs of every group, and the groups of every element list, + /// in a uniformly random order ([`order::shuffle`](super::order::shuffle)). + /// The server reads `q` as sets (`jsonb @>` over `sv` and `el`), so the + /// order changes no answer; shuffled, it no longer restates the + /// document's depth-first order. + pub(crate) fn shuffle(&mut self, rng: &mut vitaminc::random::SafeRand) { + super::order::shuffle(&mut self.sv, rng); + super::order::shuffle(&mut self.el, rng); + for group in &mut self.el { + group.shuffle(rng); + } + } + /// `jsonb @>` over two projections: every needle pair is some pair of /// this projection, and every needle element fits inside some element /// of it, recursively. What the server computes natively over the two diff --git a/packages/stack-encrypt/tests/common/mod.rs b/packages/stack-encrypt/tests/common/mod.rs index f2454edd1..c99cdc37b 100644 --- a/packages/stack-encrypt/tests/common/mod.rs +++ b/packages/stack-encrypt/tests/common/mod.rs @@ -225,13 +225,15 @@ pub async fn recording_cipher() -> (StackCipher, Arc StackCipher { - StackCipher::builder() - .kms(DeterministicSource::new(seed)) - .init() - .await - .expect("build cipher") + let builder = StackCipher::builder().kms(DeterministicSource::new(seed)); + #[cfg(feature = "test-support")] + let builder = builder.json_order_seed(seed); + builder.init().await.expect("build cipher") } /// A passthrough payload: the frame of `value`, in the clear. diff --git a/packages/stack-encrypt/tests/json_properties.proptest-regressions b/packages/stack-encrypt/tests/json_properties.proptest-regressions new file mode 100644 index 000000000..8f6641d3b --- /dev/null +++ b/packages/stack-encrypt/tests/json_properties.proptest-regressions @@ -0,0 +1,7 @@ +# Seeds for failure cases proptest has generated in the past. It is +# automatically read and these particular cases re-run before any +# novel cases are generated. +# +# It is recommended to check this file in to source control so that +# everyone who runs the test benefits from these saved cases. +cc e3fe453662b28ef6c5635a35771fdcecf5181ad878099562fbd2be196c557607 # shrinks to steps = [Index(1)], bound = 2, sealed = true From aaf5da05fd2e2cc366ff6114d50faea6498fd293 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 12:07:21 +1100 Subject: [PATCH 04/22] feat(stack-encrypt): load the keyset when opening a JSON document with its root open Under the cascade, opening a whole JSON document whose root is open (`sealed_root: false`) walks the document's chain from the root's token, as the server would for a read of `$`, so the client needs the index key of the keyset the document was sealed under. The cascade commit took it from the cipher's keyset cache and refused when the keyset was not loaded, so a document sealed under a tenant keyset failed to open through a cipher that had not selected that tenant first. The open now asks for the keyset beside the data key, as a new kind of request a Pending carries (`Request::load_keyset`). Settling answers it the way `StackCipher::keyset` selects a keyset by id: from the cache, or loaded from ZeroKMS and cached on a miss. The request is held to a pending's keyset scope like a retrieve, so a keyset cipher refuses a document of another keyset before any I/O. Settling a pending asks only for a DataKeySource, and adding IndexKeySource there would tighten public bounds across the crate and its bindings. So the cipher captures the backend's `load_index_key` at `init`, where `K: IndexKeySource` is known, as a plain function pointer returning a boxed future (Send on native, as every pending's future is), and the keyset request goes through it. No public bound changes. The sealed root (the default) opens from its root alone and loads nothing. Tested with a reader cipher over the deterministic key source that has loaded only its default keyset: it opens an open-root document of another keyset, loading that keyset once and looking it up after. Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- packages/stack-encrypt/src/cipher.rs | 60 ++++++-- packages/stack-encrypt/src/json/derive.rs | 38 +++-- packages/stack-encrypt/src/target/pending.rs | 15 +- packages/stack-encrypt/src/target/request.rs | 139 +++++++++++++++---- packages/stack-encrypt/tests/keysets.rs | 133 ++++++++++++++++++ 5 files changed, 328 insertions(+), 57 deletions(-) diff --git a/packages/stack-encrypt/src/cipher.rs b/packages/stack-encrypt/src/cipher.rs index 1515f6463..5468af0b2 100644 --- a/packages/stack-encrypt/src/cipher.rs +++ b/packages/stack-encrypt/src/cipher.rs @@ -534,6 +534,39 @@ pub struct StackCipher { /// Where the stored orders of the JSON documents this cipher seals are /// drawn from: fresh entropy per document, outside tests. json_order: crate::json::order::OrderSource, + /// The backend's [`IndexKeySource::load_index_key`], captured at + /// [`init`](StackCipherBuilder::init), where `K` is known to implement + /// it. Settling a [`Pending`](crate::target::Pending) asks only for a + /// [`DataKeySource`]; a keyset request reaches the index key through + /// this, so the settling path's bounds stay what they were. + index_keys: IndexKeyLoader, +} + +/// The future of one index-key load, `Send` on native targets as every +/// pending's future is ([`MaybeSend`](crate::kms::MaybeSend)). +#[cfg(not(target_arch = "wasm32"))] +type IndexKeyFuture<'k> = std::pin::Pin< + Box< + dyn std::future::Future> + + Send + + 'k, + >, +>; + +/// See the native definition above; identical minus the `Send` bound. +#[cfg(target_arch = "wasm32")] +type IndexKeyFuture<'k> = std::pin::Pin< + Box< + dyn std::future::Future> + + 'k, + >, +>; + +/// [`IndexKeySource::load_index_key`] for one `K`, as a plain function. +type IndexKeyLoader = for<'k> fn(&'k K, IdentifiedBy) -> IndexKeyFuture<'k>; + +fn load_index_key_of(kms: &K, keyset: IdentifiedBy) -> IndexKeyFuture<'_> { + Box::pin(kms.load_index_key(Some(keyset))) } #[cfg(feature = "http")] @@ -595,14 +628,24 @@ impl StackCipher { &self.kms } - /// The loaded state of the keyset with `id`, if this cipher holds it: - /// what opening a JSON document with its root open derives the root's - /// token with. - pub(crate) fn loaded_keyset(&self, id: Uuid) -> Option> { - match self.keysets().get(&IdentifiedBy::Uuid(id)) { - Lookup::Hit(state) => Some(state), - Lookup::Stale(_) | Lookup::Miss(_) => None, - } + /// The loaded state of the keyset `id`: from the cache, or loaded from + /// ZeroKMS and cached on a miss, exactly as [`keyset`](Self::keyset) + /// selects a keyset by id. What a keyset request settles to + /// ([`Request::load_keyset`](crate::target::Request)). + pub(crate) async fn keyset_state(&self, id: Uuid) -> Result, Error> { + // Bound first, so the cache's lock is released before the await. + let lookup = self.keysets().get(&IdentifiedBy::Uuid(id)); + let resolution = match lookup { + Lookup::Hit(state) => return Ok(state), + Lookup::Stale(resolution) | Lookup::Miss(resolution) => resolution, + }; + let (id, index_key) = (self.index_keys)(&self.kms, IdentifiedBy::Uuid(id)).await?; + let state = Arc::new(KeysetState { + id, + name: None, + prf: hmac_prf_from_index_key(&index_key), + }); + Ok(self.keysets().insert(state, resolution)) } /// Where this cipher draws a JSON document's stored order from. @@ -916,6 +959,7 @@ impl StackCipherBuilder { .map_or(crate::json::order::OrderSource::Entropy, |seed| { crate::json::order::OrderSource::controlled(seed) }), + index_keys: load_index_key_of::, }) } } diff --git a/packages/stack-encrypt/src/json/derive.rs b/packages/stack-encrypt/src/json/derive.rs index 1fc043664..d0a876767 100644 --- a/packages/stack-encrypt/src/json/derive.rs +++ b/packages/stack-encrypt/src/json/derive.rs @@ -1154,7 +1154,11 @@ fn retrieval_of( /// for a read of `$`: it derives the root's token, finds each node's /// children by their parent tags, and opens each node under the digest of /// its children's tags ([`place`]). Deriving the root's token takes the -/// index key of the keyset the document was sealed under. +/// index key of the keyset the document was sealed under, which the pending +/// asks for beside the data key ([`Request::load_keyset`]): the cipher's +/// cached keyset, or one loaded from ZeroKMS and cached, as +/// [`StackCipher::keyset`] would load it. The sealed root needs no index +/// key. fn open_document<'a, K>( doc: JsonDocument, cipher: &'a StackCipher, @@ -1166,27 +1170,21 @@ fn open_document<'a, K>( Ok(parts) => parts, Err(error) => return Pending::failed(cipher, error), }; - let keyset = if doc.header.sealed_root { - None - } else { - match cipher.loaded_keyset(doc.header.keyset_id) { - Some(state) => Some(state), - None => { - return Pending::failed( - cipher, - Error::Other( - "opening a JSON document with its root open needs its keyset loaded".into(), - ), - ) - } - } - }; - Pending::request(cipher, vec![request], move |responses| { + // With the root open, the keyset the document names is loaded beside + // its data key: from the cipher's cache, or from ZeroKMS on a miss. + let open_root = !doc.header.sealed_root; + let mut requests = vec![request]; + if open_root { + requests.push(Request::load_keyset(doc.header.keyset_id)); + } + Pending::request(cipher, requests, move |responses| { let key = responses.next_retrieved_key()?; let aes = leaf_cipher(&key)?; - match keyset { - None => open_sealed_root(&doc, &aes, &aad), - Some(keyset) => open_whole_chain(&doc, &aes, &aad, Selectors::new(keyset, prf_context)), + if open_root { + let keyset = responses.next_keyset()?; + open_whole_chain(&doc, &aes, &aad, Selectors::new(keyset, prf_context)) + } else { + open_sealed_root(&doc, &aes, &aad) } }) } diff --git a/packages/stack-encrypt/src/target/pending.rs b/packages/stack-encrypt/src/target/pending.rs index ca7f2af5b..ebdd2523d 100644 --- a/packages/stack-encrypt/src/target/pending.rs +++ b/packages/stack-encrypt/src/target/pending.rs @@ -189,7 +189,7 @@ impl<'a, T: 'a, K> Pending<'a, T, K> { where F: FnOnce(&mut Responses) -> Result + MaybeSend + 'a, { - let (generated, retrieved) = tally(&requests); + let (generated, retrieved, keysets) = tally(&requests); if let Err(error) = check_scope(scope.keyset(), &requests) { return Self::failed(scope, error); } @@ -199,7 +199,7 @@ impl<'a, T: 'a, K> Pending<'a, T, K> { requests, failed: None, fulfil: Box::new(move |responses| { - let mut own = responses.split_front(generated, retrieved)?; + let mut own = responses.split_front(generated, retrieved, keysets)?; let value = fulfil(&mut own)?; if !own.is_exhausted() { return Err(Error::ResponseShape); @@ -474,9 +474,11 @@ async fn dispatch( ) -> Result { let mut generates: Vec = Vec::new(); let mut retrieves: Vec = Vec::new(); + let mut loads: Vec = Vec::new(); for request in requests { match request.into_kind() { RequestKind::GenerateDataKey { descriptor } => generates.push(descriptor), + RequestKind::LoadKeyset { keyset_id } => loads.push(keyset_id), RequestKind::RetrieveDataKey { iv, tag, @@ -574,7 +576,14 @@ async fn dispatch( .map(|key| key.ok_or(Error::ResponseShape)) .collect::>()?; - Ok(Responses::new(generated, retrieved)) + // A keyset request is answered from the cipher's cache, and loaded from + // ZeroKMS only on a miss, as selecting the keyset would load it. + let mut keysets = Vec::with_capacity(loads.len()); + for keyset_id in loads { + keysets.push(cipher.keyset_state(keyset_id).await?); + } + + Ok(Responses::new(generated, retrieved).with_keysets(keysets)) } #[cfg(test)] diff --git a/packages/stack-encrypt/src/target/request.rs b/packages/stack-encrypt/src/target/request.rs index 10257b69d..3dadb7a33 100644 --- a/packages/stack-encrypt/src/target/request.rs +++ b/packages/stack-encrypt/src/target/request.rs @@ -13,15 +13,18 @@ use super::context::AeadContext; use std::collections::VecDeque; +use std::sync::Arc; use stack_kms::{DataKey, DataKeyWithTag, Iv}; use uuid::Uuid; +use crate::keyset::KeysetState; use crate::{Descriptor, Error}; /// One unit of ZeroKMS work a [`Pending`](super::Pending) needs: /// constructible, otherwise opaque, so new request kinds (a PRF derivation, a -/// keyset override) can be added without breaking implementations. +/// keyset override, a keyset's index key) can be added without breaking +/// implementations. #[derive(Debug, Clone)] pub struct Request(RequestKind); @@ -39,6 +42,12 @@ pub(super) enum RequestKind { descriptor: Descriptor, keyset_id: Uuid, }, + /// The loaded state of the keyset `keyset_id`, its index-key PRF: from + /// the cipher's keyset cache, or loaded from ZeroKMS and cached on a + /// miss, as [`StackCipher::keyset`](crate::StackCipher::keyset) loads + /// one. What opening a JSON document with its root open derives the + /// root's token with. + LoadKeyset { keyset_id: Uuid }, } impl Request { @@ -106,32 +115,41 @@ impl Request { }) } + /// Request the loaded state of the keyset `keyset_id` (crate-internal: + /// the state holds the keyset's index-key PRF, which never leaves the + /// crate). + pub(crate) fn load_keyset(keyset_id: Uuid) -> Self { + Self(RequestKind::LoadKeyset { keyset_id }) + } + /// Consume the request, yielding what it asks for. pub(super) fn into_kind(self) -> RequestKind { self.0 } - /// The keyset a retrieve request names; `None` for a generate request, - /// which mints under the pending's keyset. + /// The keyset a retrieve or keyset request names; `None` for a generate + /// request, which mints under the pending's keyset. pub(super) fn retrieve_keyset(&self) -> Option { match &self.0 { RequestKind::GenerateDataKey { .. } => None, - RequestKind::RetrieveDataKey { keyset_id, .. } => Some(*keyset_id), + RequestKind::RetrieveDataKey { keyset_id, .. } + | RequestKind::LoadKeyset { keyset_id } => Some(*keyset_id), } } } /// How many requests of each kind `requests` holds, as -/// `(generate, retrieve)` — the shape a fulfilment is scoped to. -pub(super) fn tally(requests: &[Request]) -> (usize, usize) { - let (mut generate, mut retrieve) = (0usize, 0usize); +/// `(generate, retrieve, keyset)` — the shape a fulfilment is scoped to. +pub(super) fn tally(requests: &[Request]) -> (usize, usize, usize) { + let (mut generate, mut retrieve, mut keyset) = (0usize, 0usize, 0usize); for request in requests { match request.0 { RequestKind::GenerateDataKey { .. } => generate += 1, RequestKind::RetrieveDataKey { .. } => retrieve += 1, + RequestKind::LoadKeyset { .. } => keyset += 1, } } - (generate, retrieve) + (generate, retrieve, keyset) } /// The responses a fulfilment draws from — one queue per request kind, in @@ -141,6 +159,7 @@ pub(super) fn tally(requests: &[Request]) -> (usize, usize) { pub struct Responses { generated: VecDeque, retrieved: VecDeque, + keysets: VecDeque>, } impl Responses { @@ -149,9 +168,17 @@ impl Responses { Self { generated: generated.into(), retrieved: retrieved.into(), + keysets: VecDeque::new(), } } + /// These responses with the keysets loaded for the keyset requests, in + /// request order. + pub(super) fn with_keysets(mut self, keysets: Vec>) -> Self { + self.keysets = keysets.into(); + self + } + /// The next generated data key, in [`Request::generate_data_key`] order. pub fn next_generated_key(&mut self) -> Result { self.generated.pop_front().ok_or(Error::ResponseShape) @@ -162,7 +189,12 @@ impl Responses { self.retrieved.pop_front().ok_or(Error::ResponseShape) } - /// Split off the first `generated` + `retrieved` responses — the + /// The next loaded keyset, in [`Request::load_keyset`] order. + pub(crate) fn next_keyset(&mut self) -> Result, Error> { + self.keysets.pop_front().ok_or(Error::ResponseShape) + } + + /// Split off the first `generated` + `retrieved` + `keysets` responses — the /// per-fulfilment view [`Pending::request`](super::Pending::request) /// scopes each fulfilment to. Short of either count is /// [`Error::ResponseShape`], and nothing is consumed. @@ -170,13 +202,18 @@ impl Responses { &mut self, generated: usize, retrieved: usize, + keysets: usize, ) -> Result { - if self.generated.len() < generated || self.retrieved.len() < retrieved { + if self.generated.len() < generated + || self.retrieved.len() < retrieved + || self.keysets.len() < keysets + { return Err(Error::ResponseShape); } Ok(Responses { generated: self.generated.drain(..generated).collect(), retrieved: self.retrieved.drain(..retrieved).collect(), + keysets: self.keysets.drain(..keysets).collect(), }) } @@ -184,7 +221,7 @@ impl Responses { /// fulfilment returns: a fulfilment that asked for a key and left it /// behind is a composition bug, not a cheaper request. pub(super) fn is_exhausted(&self) -> bool { - self.generated.is_empty() && self.retrieved.is_empty() + self.generated.is_empty() && self.retrieved.is_empty() && self.keysets.is_empty() } pub(crate) fn drain_generated(&mut self) -> impl Iterator + '_ { @@ -252,19 +289,20 @@ mod tests { #[test] fn tally_counts_nothing_for_no_requests() { - assert_eq!(tally(&[]), (0, 0)); + assert_eq!(tally(&[]), (0, 0, 0)); } #[test] - fn tally_separates_the_two_kinds() { + fn tally_separates_the_three_kinds() { let requests = vec![ Request::generate_under(d()), Request::retrieve_under(Iv::default(), vec![1], d(), ks()), Request::generate_under(d()), Request::retrieve_under(Iv::default(), vec![2], d(), ks()), Request::generate_under(d()), + Request::load_keyset(ks()), ]; - assert_eq!(tally(&requests), (3, 2)); + assert_eq!(tally(&requests), (3, 2, 1)); } /// The public constructors render the descriptor themselves, from the @@ -277,11 +315,11 @@ mod tests { let expected = Descriptor::of(context); match Request::generate_data_key(context).into_kind() { RequestKind::GenerateDataKey { descriptor } => assert_eq!(descriptor, expected), - RequestKind::RetrieveDataKey { .. } => panic!("expected a generate request"), + _ => panic!("expected a generate request"), } match Request::retrieve_data_key(Iv::default(), vec![1], context, ks()).into_kind() { RequestKind::RetrieveDataKey { descriptor, .. } => assert_eq!(descriptor, expected), - RequestKind::GenerateDataKey { .. } => panic!("expected a retrieve request"), + _ => panic!("expected a retrieve request"), } } @@ -309,13 +347,13 @@ mod tests { let context = NonEmpty::new(Tenant("acme".into())).unwrap(); match Request::generate_data_key(context.clone()).into_kind() { RequestKind::GenerateDataKey { descriptor } => assert_eq!(descriptor.as_str(), "acme"), - RequestKind::RetrieveDataKey { .. } => panic!("expected a generate request"), + _ => panic!("expected a generate request"), } match Request::retrieve_data_key(Iv::default(), vec![1], context, ks()).into_kind() { RequestKind::RetrieveDataKey { descriptor, .. } => { assert_eq!(descriptor.as_str(), "acme") } - RequestKind::GenerateDataKey { .. } => panic!("expected a retrieve request"), + _ => panic!("expected a retrieve request"), } } @@ -323,7 +361,7 @@ mod tests { fn a_generate_request_carries_its_descriptor() { match Request::generate_under(d()).into_kind() { RequestKind::GenerateDataKey { descriptor } => assert_eq!(descriptor, d()), - RequestKind::RetrieveDataKey { .. } => panic!("expected a generate request"), + _ => panic!("expected a generate request"), } } @@ -342,7 +380,7 @@ mod tests { assert_eq!(tag, vec![7, 8, 9]); assert_eq!(descriptor, d()); } - RequestKind::GenerateDataKey { .. } => panic!("expected a retrieve request"), + _ => panic!("expected a retrieve request"), } } @@ -408,7 +446,7 @@ mod tests { #[tokio::test] async fn split_front_takes_exactly_what_was_asked_for() { let mut responses = responses(3, 2).await; - let own = responses.split_front(2, 1).unwrap(); + let own = responses.split_front(2, 1, 0).unwrap(); assert_eq!((own.generated.len(), own.retrieved.len()), (2, 1)); assert_eq!( @@ -425,10 +463,10 @@ mod tests { let tags: Vec> = generated.iter().map(|key| key.tag.clone()).collect(); let mut responses = Responses::new(generated, Vec::new()); - let mut first = responses.split_front(1, 0).unwrap(); + let mut first = responses.split_front(1, 0, 0).unwrap(); assert_eq!(first.next_generated_key().unwrap().tag, tags[0]); - let mut rest = responses.split_front(2, 0).unwrap(); + let mut rest = responses.split_front(2, 0, 0).unwrap(); assert_eq!(rest.next_generated_key().unwrap().tag, tags[1]); assert_eq!(rest.next_generated_key().unwrap().tag, tags[2]); } @@ -436,7 +474,7 @@ mod tests { #[tokio::test] async fn split_front_of_nothing_yields_an_empty_view() { let mut responses = responses(2, 2).await; - let mut own = responses.split_front(0, 0).unwrap(); + let mut own = responses.split_front(0, 0, 0).unwrap(); assert!(matches!( own.next_generated_key(), @@ -457,7 +495,7 @@ mod tests { async fn split_front_short_of_generated_responses_errors_without_consuming() { let mut responses = responses(1, 0).await; assert!(matches!( - responses.split_front(2, 0), + responses.split_front(2, 0, 0), Err(Error::ResponseShape) )); assert_eq!(responses.generated.len(), 1); @@ -467,7 +505,7 @@ mod tests { async fn split_front_short_of_retrieved_responses_errors_without_consuming() { let mut responses = responses(2, 1).await; assert!(matches!( - responses.split_front(2, 2), + responses.split_front(2, 2, 0), Err(Error::ResponseShape) )); // Neither queue was drained: the check happens before the split. @@ -494,4 +532,53 @@ mod tests { Err(Error::ResponseShape) )); } + + fn keyset(id: u128) -> Arc { + use vitaminc_prf::PrfKeyInit; + Arc::new(KeysetState { + id: Uuid::from_u128(id), + name: None, + prf: vitaminc_hmac::HmacSha256Prf::new(vitaminc_protected::Protected::new([1; 32])), + }) + } + + #[test] + fn a_keyset_request_names_its_keyset() { + let request = Request::load_keyset(ks()); + assert_eq!(request.retrieve_keyset(), Some(ks()), "held to the scope"); + match request.into_kind() { + RequestKind::LoadKeyset { keyset_id } => assert_eq!(keyset_id, ks()), + _ => panic!("expected a keyset request"), + } + } + + #[test] + fn keysets_come_back_in_request_order_and_not_past_the_end() { + let mut responses = + Responses::new(Vec::new(), Vec::new()).with_keysets(vec![keyset(1), keyset(2)]); + assert!(!responses.is_exhausted()); + assert_eq!(responses.next_keyset().unwrap().id, Uuid::from_u128(1)); + assert_eq!(responses.next_keyset().unwrap().id, Uuid::from_u128(2)); + assert!(responses.is_exhausted()); + assert!(matches!(responses.next_keyset(), Err(Error::ResponseShape))); + } + + #[test] + fn split_front_takes_keysets_and_refuses_short_of_them() { + let mut responses = Responses::new(Vec::new(), Vec::new()).with_keysets(vec![ + keyset(1), + keyset(2), + keyset(3), + ]); + assert!(matches!( + responses.split_front(0, 0, 4), + Err(Error::ResponseShape) + )); + assert_eq!(responses.keysets.len(), 3, "nothing consumed"); + let mut own = responses.split_front(0, 0, 2).unwrap(); + assert_eq!(own.next_keyset().unwrap().id, Uuid::from_u128(1)); + assert_eq!(own.next_keyset().unwrap().id, Uuid::from_u128(2)); + assert_eq!(responses.keysets.len(), 1, "the sibling's keyset is left"); + assert!(!responses.is_exhausted()); + } } diff --git a/packages/stack-encrypt/tests/keysets.rs b/packages/stack-encrypt/tests/keysets.rs index f36238188..3a4ea84ee 100644 --- a/packages/stack-encrypt/tests/keysets.rs +++ b/packages/stack-encrypt/tests/keysets.rs @@ -712,3 +712,136 @@ async fn a_mixed_keyset_column_does_not_open_through_a_keyset_handle() { "refused before any key was retrieved" ); } + +// ============================================================================= +// A JSON document with its root open +// ============================================================================= + +/// The deterministic source, which re-derives a data key in any process, +/// with a count of keyset loads: what a reader in another process holds. +struct Reader { + inner: stack_kms::DeterministicSource, + loads: AtomicUsize, +} + +impl Reader { + fn new() -> Self { + Self { + inner: stack_kms::DeterministicSource::new([7; 32]), + loads: AtomicUsize::new(0), + } + } + + fn loads(&self) -> usize { + self.loads.load(Ordering::Relaxed) + } +} + +impl DataKeySource for Reader { + async fn generate_keys( + &self, + payloads: Vec>, + keyset_id: Option, + unverified_context: Option>, + ) -> Result, stack_kms::Error> { + self.inner + .generate_keys(payloads, keyset_id, unverified_context) + .await + } + + async fn retrieve_keys( + &self, + payloads: Vec>, + keyset_id: Option, + unverified_context: Option<&UnverifiedContext>, + ) -> Result, stack_kms::Error> { + self.inner + .retrieve_keys(payloads, keyset_id, unverified_context) + .await + } +} + +impl IndexKeySource for Reader { + async fn load_index_key( + &self, + keyset_id: Option, + ) -> Result<(Uuid, IndexKey), stack_kms::Error> { + let _ = self.loads.fetch_add(1, Ordering::Relaxed); + self.inner.load_index_key(keyset_id).await + } +} + +async fn reader() -> StackCipher { + StackCipher::builder() + .kms(Reader::new()) + .init() + .await + .expect("build cipher") +} + +/// Opening a JSON document whose root is open walks its chain from the +/// root's token, which takes the index key of the keyset it was sealed +/// under. A cipher that has not loaded that keyset loads it, once, as +/// selecting it would, and opens the document; after that it is a lookup. +/// A document with its root sealed opens from its root alone and loads +/// nothing. +#[tokio::test] +async fn opening_an_open_root_json_document_loads_its_keyset_once() { + use serde_json::json; + use stack_encrypt::json::{JsonDocument, JsonOptions}; + use stack_encrypt::target::{json_document, AeadContext, Borrowed, CallerContext}; + + let value = json!({"a": [1, {"b": "two"}], "c": null}); + let seal = |sealed_root: bool| { + let value = value.clone(); + async move { + let writer = reader().await; + let tenant = writer.keyset(name("acme")).await.expect("select"); + tenant + .run( + json_document::<_, Borrowed>(JsonOptions { + sealed_root, + ..JsonOptions::new() + }), + &value, + CallerContext::from(nonempty!("users").with("profile")), + ) + .await + .expect("seals") + } + }; + let aead = || AeadContext::from(nonempty!("users").with("profile")); + + let reader = reader().await; + assert_eq!(reader.kms().loads(), 1, "the reader has its default only"); + let sealed: JsonDocument = seal(true).await; + let opened: serde_json::Value = sealed + .decrypt_into(&reader, aead()) + .await + .expect("a sealed root opens"); + assert_eq!(opened, value); + assert_eq!(reader.kms().loads(), 1, "a sealed root needs no index key"); + + let open: JsonDocument = seal(false).await; + for _ in 0..2 { + let opened: serde_json::Value = open + .clone() + .decrypt_into(&reader, aead()) + .await + .expect("an open root opens through a cipher that had not loaded its keyset"); + assert_eq!(opened, value); + } + assert_eq!( + reader.kms().loads(), + 2, + "the document's keyset loads on the first open and is cached after" + ); + let acme = reader.keyset(name("acme")).await.expect("select"); + assert_eq!(acme.keyset_id(), open.header().keyset_id()); + let opened: serde_json::Value = open + .clone() + .decrypt_into(&acme, aead()) + .await + .expect("and through the keyset's own cipher"); + assert_eq!(opened, value); +} From 9cc58274c09994547bebacddbc3858462e769364 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 13:06:22 +1100 Subject: [PATCH 05/22] test(stack-encrypt): find JSON rows by path, and assert where each refusal comes from With the entries stored in a random order, a test can no longer name a node by its depth-first slot. Every test that did now finds the row by its path, through the self tag of the path's token as the server does: the hostile-row tests (tests/json_read_checks.rs), tests/json_index.rs, tests/json_properties.rs, and the PostgreSQL oracle (packages/eql/tests/encryption/tests/json_tree.rs, whose two slot assertions now compare as sets, since the SQL returns a token's rows by slot). `q` and the selector lists compare as sets. Each hostile-row test asserts the error the new code produces, not either of two. Beneath the node a read names, a swapped entry, a dropped child, an added child, reordered children, a child moved under another parent, a truncated subtree and another column are refused by the cipher (Aead); a repeated row, a row no read reaches, a wrong depth, a self tag no rule derives (or not the one the label names), the wrong selector or label at the top of a read, the sealed root out of place and a wrong child count are JsonWrongEntry. Tests that forge rows compute the chain's public hashes the way a database holding the read's tokens can. New properties: a read's rows open to the same value in any order; any one row of a whole-document read changed, dropped, repeated or misreported is refused with the error that names how; the stored order is a permutation with the root at slot 0, and every seal draws its own. The pinned vectors (tests/json_tree_vectors.rs, behind `test-support`) now hold, per root option, the entries in their stored order, the path stored at each slot (the permutation the seeded generator draws), the selector multiset, `q` as sets and the tokens. The tags, the iv and the tokens are unchanged from the previous pin; only the order moved. The leakage profile (tests/json_leakage) models the random order: a new `StoredOrder` clause, a simulator that draws the same permutation, and a canonical order (canonical.rs) that every distinguisher reads the world in, built only from what an observer sees without the stored order. Distinguishers that read the depth-first order are retired; a mutant claiming that order is caught by `walked_subtrees_are_contiguous`. It found that a containment needle is still sent in key byte order, which the leakage page now states. Rewriting the tests also tightened the placement in derive.rs: rows of two documents are refused before repeated slots (two documents' entries can share a slot); a re-entry check that no key's disjoint paths can reach is gone; and unit tests cover the whole-document chain walk, a forged child count and another column. Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- .../eql/tests/encryption/tests/json_tree.rs | 220 ++++-- packages/stack-encrypt/src/json/derive.rs | 152 ++++- .../tests/fixtures/json_tree_vectors.json | 172 ++++- packages/stack-encrypt/tests/json_index.rs | 317 ++++++--- .../tests/json_leakage/calibration.rs | 115 ++-- .../tests/json_leakage/canonical.rs | 637 ++++++++++++++++++ .../tests/json_leakage/corpus.rs | 14 +- .../tests/json_leakage/distinguishers.rs | 423 +++++------- .../stack-encrypt/tests/json_leakage/mod.rs | 145 ++-- .../tests/json_leakage/mutants.rs | 90 +-- .../tests/json_leakage/observe.rs | 22 +- .../tests/json_leakage/profile.rs | 101 ++- .../tests/json_leakage/simulator.rs | 72 +- .../stack-encrypt/tests/json_properties.rs | 217 +++++- .../stack-encrypt/tests/json_read_checks.rs | 616 +++++++++++++---- .../stack-encrypt/tests/json_tree_vectors.rs | 83 ++- 16 files changed, 2572 insertions(+), 824 deletions(-) create mode 100644 packages/stack-encrypt/tests/json_leakage/canonical.rs diff --git a/packages/eql/tests/encryption/tests/json_tree.rs b/packages/eql/tests/encryption/tests/json_tree.rs index fe4ea4f19..6ae1431a9 100644 --- a/packages/eql/tests/encryption/tests/json_tree.rs +++ b/packages/eql/tests/encryption/tests/json_tree.rs @@ -69,6 +69,18 @@ fn options() -> JsonOptions { } /// Padded standard base64, as the wire spells a selector. +/// A projection with every list sorted: `q` as the sets `jsonb @>` reads. +fn as_set(q: &Value) -> Value { + let mut sv = q["sv"].as_array().cloned().unwrap_or_default(); + sv.sort_by_key(|pair| pair.to_string()); + let mut el: Vec = q["el"] + .as_array() + .map(|groups| groups.iter().map(as_set).collect()) + .unwrap_or_default(); + el.sort_by_key(|group| group.to_string()); + json!({"sv": sv, "el": el}) +} + fn base64(bytes: &[u8]) -> String { use base64::Engine; base64::engine::general_purpose::STANDARD.encode(bytes) @@ -123,8 +135,10 @@ async fn a_document_round_trips_through_the_typed_target() { ); // An entry lifted from the stored document, with the header and its - // ordinal grafted as `->` grafts them, opens to its scalar through the - // selector that asked for it, and only through that one. + // slot grafted as `->` grafts them, opens to its scalar through the + // selector that asked for it, and only through that one. The entries + // are stored in a random order: `$.hello` is the one entry carrying its + // selector. let keyset = cipher.default_keyset(); let selector = |path: &str| { let path = JsonPath::parse(path).unwrap(); @@ -132,9 +146,14 @@ async fn a_document_round_trips_through_the_typed_target() { async move { keyset.json_selector(&path, column()).await.unwrap() } }; let hello = selector("$.hello").await; - let mut entry = stored.sv[1].clone(); + let at = stored + .sv + .iter() + .position(|e| serde_json::to_value(&e.s).unwrap() == json!(hello.to_string())) + .expect("the document has $.hello"); + let mut entry = stored.sv[at].clone(); entry.h = Some(stored.h.clone()); - entry.j = Some(1); + entry.j = Some(at as u64); entry.i = Some(stored.i.clone()); entry.v = Some(stored.v); let scalar: Value = cipher @@ -161,7 +180,7 @@ async fn a_document_round_trips_through_the_typed_target() { ), "another path's selector refuses the entry: {refused:?}" ); - let bare = stored.sv[1].clone(); + let bare = stored.sv[at].clone(); assert!(cipher .decrypt_as::(JsonTreeExtracted::from_selector(&hello, bare), column()) .await @@ -197,16 +216,24 @@ async fn the_query_operands_are_the_engines_terms_in_the_sql_contract() { assert_eq!(leaf.s.0.as_str(), bound.selector().to_string()); // The engine's document under the label and the EQL document under the - // identifier carry the same selectors: one context, two spellings. + // identifier carry the same selectors: one context, two spellings. Each + // is stored in its own random order, so they compare as multisets, and + // the projections as the sets `jsonb @>` reads. let engine = cipher.encrypt(&fixture()).using(&plan).await.unwrap(); let typed = seal(&cipher, &fixture()).await; assert_eq!(engine.entries().len(), typed.sv.len()); - for (a, b) in engine.entries().iter().zip(&typed.sv) { - assert_eq!(base64(a.selector()), b.s.0.as_str()); - } + let mut a: Vec = engine + .entries() + .iter() + .map(|e| base64(e.selector())) + .collect(); + let mut b: Vec = typed.sv.iter().map(|e| e.s.0.as_str().to_owned()).collect(); + a.sort_unstable(); + b.sort_unstable(); + assert_eq!(a, b, "the same selectors"); assert_eq!( - serde_json::to_value(engine.projection()).unwrap(), - serde_json::to_value(&typed.q).unwrap(), + as_set(&serde_json::to_value(engine.projection()).unwrap()), + as_set(&serde_json::to_value(&typed.q).unwrap()), "and the same projection" ); let opened: Value = cipher.open(engine).using(&plan).await.unwrap(); @@ -578,8 +605,9 @@ fn postgres_extracts_walks_and_orders_by_path() { assert_eq!(exists, [1, 2]); // Every element of an array shares its selector: the path query returns - // one row per element, each opening to its value. - let tags: Vec = db + // one row per element, each opening to its value, in the stored order, + // which is random: the client recovers element order from the labels. + let mut tags: Vec = db .query( "SELECT e::jsonb::text FROM json_tree_rows, eql_v4.json_tree_path_query(doc::jsonb, $1::text) e WHERE id = 3", &[&selector("$.tags[*]")], @@ -588,7 +616,30 @@ fn postgres_extracts_walks_and_orders_by_path() { .iter() .map(|r| open(&selector_of("$.tags[*]"), entry(r))) .collect(); + tags.sort_by_key(Value::to_string); assert_eq!(tags, vec![json!("x"), json!("y")]); + // `->` at a wildcard path takes the first entry carrying the selector + // in the stored order: an unspecified element of the row, the same one + // every time, and one of the elements `jsonb_path_query` returns. + let first: Vec = (0..3) + .map(|_| { + let row = db + .query_one( + "SELECT (doc -> $1::text)::jsonb::text FROM json_tree_rows WHERE id = 3", + &[&selector("$.tags[*]")], + ) + .unwrap(); + open(&selector_of("$.tags[*]"), entry(&row)) + }) + .collect(); + assert!( + tags.contains(&first[0]), + "`->` at a wildcard is one of the elements" + ); + assert!( + first.iter().all(|v| v == &first[0]), + "the same one every time" + ); // The read of `$.nested`, from a token derived on the client from the // path alone: nested itself and its six entries in row 1, reassembling @@ -741,9 +792,26 @@ fn postgres_extracts_walks_and_orders_by_path() { .iter() .map(entry) .collect(); + // `json_tree_children` returns the rows in the stored order. + let mut found: Vec<(Option, Option)> = elements.iter().map(|e| (e.j, e.d)).collect(); + found.sort_unstable(); + let first = slot( + &mut db, + "json_tree_rows", + 1, + &subtree_key("$.nested.list[0]"), + ); + let second = slot( + &mut db, + "json_tree_rows", + 1, + &subtree_key("$.nested.list[1]"), + ); + let mut expected = vec![(Some(first), Some(1)), (Some(second), Some(1))]; + expected.sort_unstable(); assert_eq!( - elements.iter().map(|e| (e.j, e.d)).collect::>(), - [(Some(5), Some(1)), (Some(7), Some(1))] + found, expected, + "both elements, each at the slot it is stored at" ); let members: i32 = db .query_one( @@ -1275,6 +1343,23 @@ fn store( } } +/// The slot the node `key` names is stored at in row `id`: the entries are +/// stored in a random order, so a test finds a node by its self tag, as the +/// server does. +fn slot(db: &mut postgres::Client, table: &str, id: i32, key: &JsonSubtreeKey) -> u64 { + let j: i64 = db + .query_one( + &format!( + "SELECT (e.o - 1)::bigint FROM {table}, jsonb_array_elements(doc::jsonb -> 'sv') WITH ORDINALITY AS e(v, o) + WHERE id = $2 AND e.v ->> 'x' = eql_v4_internal.json_tree_self_tag(eql_v4_internal.json_tree_iv(doc::jsonb ->> 'h'), decode($1::text, 'base64'))" + ), + &[&key.tokens()[0], &id], + ) + .unwrap_or_else(|error| panic!("{table} {} in row {id}: {error}", key.path())) + .get(0); + j as u64 +} + /// The rows `json_tree_subtree(doc, key.tokens(), budget)` returns in row `id`. fn subtree_rows( db: &mut postgres::Client, @@ -1836,12 +1921,14 @@ fn postgres_a_position_reaches_array_elements_only() { /// (another path's entry, the sealed root, another position's subtree with /// its self tag forged) and from stored rows tampered with (a self tag /// forged so the read finds the wrong node, a top-level field moved under -/// another node, two entries swapped in `sv`). Each is refused: the wrong -/// entry as `JsonWrongEntry`, a moved entry as `Aead`, since its ciphertext -/// is bound to the position it was sealed at. Two answers are allowed by -/// design and shown to open: another row's node at the path (ADR 0010), -/// and the node at the same selector under another element, which no field -/// a row binds tells apart. +/// another node, two entries swapped in `sv`). Each is refused: an entry +/// that is not the one asked for at the top of a read as `JsonWrongEntry`; +/// anything changed beneath the node a read names as `Aead`, since every +/// node's ciphertext commits to its children's tags; and a moved entry as +/// `Aead`, since its ciphertext is bound to the slot it was sealed at. Two +/// answers are allowed by design and shown to open: another row's node at +/// the path (ADR 0010), and the node at the same selector under another +/// element at the top of a read, which nothing the node binds tells apart. #[test] #[ignore = "requires EQL_TEST_DATABASE_URL pointing to an EQL v4 database"] fn postgres_hostile_answers_are_refused_by_the_client() { @@ -1942,16 +2029,23 @@ fn postgres_hostile_answers_are_refused_by_the_client() { ); // The read of `$.nested.list[1]` answered with `$.nested.list[0]`'s - // rows, its self tag forged to the second element's: the same selector, - // and the position in its label tells them apart. + // rows, its self tag forged to the second element's: the chain finds + // none of the second element's children among them, and the first + // element, sealed over its own `x`, does not open. let second = subtree_key("$.nested.list[1]"); assert_eq!( opened(&second, read(&mut db, 1, &second)).unwrap(), json!({"x": 2}) ); let mut forged = read(&mut db, 1, &subtree_key("$.nested.list[0]")); - forged[0].x = serde_json::from_value(json!(x_of(&mut db, 1, &second))).unwrap(); - wrong(opened(&second, forged), "another position's subtree"); + // The read returns its rows by slot; the element itself is at depth 0. + let top = forged.iter().position(|e| e.d == Some(0)).unwrap(); + forged[top].x = serde_json::from_value(json!(x_of(&mut db, 1, &second))).unwrap(); + let refused = opened(&second, forged); + assert!( + matches!(refused, Err(Error::Aead)), + "another position's subtree: {refused:?}" + ); // Allowed: `$.nested.list[0].x` for `$.nested.list[1].x`, self tag // forged; the same selector, the same label. @@ -1977,40 +2071,62 @@ fn postgres_hostile_answers_are_refused_by_the_client() { assert!(opened(&nested, truncated).is_err(), "a truncated read"); let mut reordered = read(&mut db, 1, &nested); reordered.swap(2, 3); + reordered.reverse(); assert_eq!( opened(&nested, reordered).unwrap(), fixture()["nested"], - "the rows are placed by their positions, not the order they came in" + "the rows are placed by the chain, not by the order they came in" ); - // A stored row whose `$.number` (entry 9) is moved under `$.nested`: - // a parent tag at the ordinal after its two members and a member's self - // tag. The read takes it for nested's third member and returns it after - // nested's subtree; nested's child count says two, and the row is - // refused. + // A stored row whose `$.number` is moved under `$.nested`: a parent tag + // at the ordinal after its two members and a member's self tag. The + // read takes it for nested's third member and returns it after nested's + // subtree; nested, sealed over the tags of its two members, does not + // open. + let number = subtree_key("$.number"); + let number_at = slot(&mut db, "json_tree_hostile", 1, &number); db.execute( - "UPDATE json_tree_hostile SET doc = jsonb_set(jsonb_set(doc::jsonb, '{sv,9,p}', - to_jsonb(eql_v4_internal.json_tree_parent_tag( - eql_v4_internal.json_tree_iv(doc::jsonb ->> 'h'), decode($1::text, 'base64'), 2))), - '{sv,9,x}', - to_jsonb(eql_v4_internal.json_tree_self_tag( - eql_v4_internal.json_tree_iv(doc::jsonb ->> 'h'), - eql_v4_internal.json_tree_member_token(decode($1::text, 'base64'), decode(doc::jsonb -> 'sv' -> 9 ->> 's', 'base64')))))::public.eql_v4_json_tree - WHERE id = 1", + &format!( + "UPDATE json_tree_hostile SET doc = jsonb_set(jsonb_set(doc::jsonb, '{{sv,{number_at},p}}', + to_jsonb(eql_v4_internal.json_tree_parent_tag( + eql_v4_internal.json_tree_iv(doc::jsonb ->> 'h'), decode($1::text, 'base64'), 2))), + '{{sv,{number_at},x}}', + to_jsonb(eql_v4_internal.json_tree_self_tag( + eql_v4_internal.json_tree_iv(doc::jsonb ->> 'h'), + eql_v4_internal.json_tree_member_token(decode($1::text, 'base64'), decode(doc::jsonb -> 'sv' -> {number_at} ->> 's', 'base64')))))::public.eql_v4_json_tree + WHERE id = 1" + ), &[&nested.tokens()[0]], ) .unwrap(); let rows = read(&mut db, 1, &nested); assert_eq!(rows.len(), 8, "the read took number for a member"); - assert_eq!(rows.last().map(|e| (e.j, e.d)), Some((Some(9), Some(1)))); - wrong(opened(&nested, rows), "a top-level field as a member"); + assert!( + rows.iter() + .any(|e| (e.j, e.d) == (Some(number_at), Some(1))), + "number among nested's members" + ); + let refused = opened(&nested, rows); + assert!( + matches!(refused, Err(Error::Aead)), + "a top-level field as a member: {refused:?}" + ); - // A stored row whose `$.hello` (1) carries `$.number`'s self tag: the - // read of `$.number` finds hello, whose selector is not number's. - let number = subtree_key("$.number"); + // A stored row whose `$.hello` carries `$.number`'s self tag: the read + // of `$.number` finds hello, whose selector is not number's. + let hello_at = slot(&mut db, "json_tree_hostile", 2, &subtree_key("$.hello")); + let number_at = slot(&mut db, "json_tree_hostile", 2, &number); + let deep_at = slot( + &mut db, + "json_tree_hostile", + 2, + &subtree_key("$.nested.deep"), + ); db.execute( - "UPDATE json_tree_hostile SET doc = jsonb_set(jsonb_set(doc::jsonb, '{sv,1,x}', doc::jsonb -> 'sv' -> 9 -> 'x'), - '{sv,9,x}', '\"AAAAAAAAAAA=\"')::public.eql_v4_json_tree WHERE id = 2", + &format!( + "UPDATE json_tree_hostile SET doc = jsonb_set(jsonb_set(doc::jsonb, '{{sv,{hello_at},x}}', doc::jsonb -> 'sv' -> {number_at} -> 'x'), + '{{sv,{number_at},x}}', '\"AAAAAAAAAAA=\"')::public.eql_v4_json_tree WHERE id = 2" + ), &[], ) .unwrap(); @@ -2019,12 +2135,14 @@ fn postgres_hostile_answers_are_refused_by_the_client() { "a forged self tag", ); - // A stored row whose `deep` (3) and `number` (9) swapped places: each - // entry is found where it now sits, and fails to open there. + // A stored row whose `deep` and `number` swapped places: each entry is + // found where it now sits, and fails to open there. db.execute( - "UPDATE json_tree_hostile SET doc = jsonb_set(jsonb_set(doc::jsonb, - '{sv,3}', doc::jsonb -> 'sv' -> 9), '{sv,9}', doc::jsonb -> 'sv' -> 3)::public.eql_v4_json_tree - WHERE id = 2", + &format!( + "UPDATE json_tree_hostile SET doc = jsonb_set(jsonb_set(doc::jsonb, + '{{sv,{deep_at}}}', doc::jsonb -> 'sv' -> {number_at}), '{{sv,{number_at}}}', doc::jsonb -> 'sv' -> {deep_at})::public.eql_v4_json_tree + WHERE id = 2" + ), &[], ) .unwrap(); diff --git a/packages/stack-encrypt/src/json/derive.rs b/packages/stack-encrypt/src/json/derive.rs index d0a876767..eb1124f67 100644 --- a/packages/stack-encrypt/src/json/derive.rs +++ b/packages/stack-encrypt/src/json/derive.rs @@ -1253,11 +1253,11 @@ fn open_whole_chain( aad, aes, }; - let mut values = place(rows, &reads, &selectors, 0, &opening)?; - match (values.pop(), values.is_empty()) { - (Some(value), true) => Ok(hand_over(value)), - _ => Err(shape("a document whose chain reaches its root")), - } + // One read, of the root the chain was seen to enter: one value. + place(rows, &reads, &selectors, 0, &opening)? + .pop() + .map(hand_over) + .ok_or_else(|| shape("a document whose chain reaches its root")) } impl DecryptInto for JsonDocument { @@ -1432,23 +1432,24 @@ fn open_subtree<'a, K>( context: AeadContext, ) -> Pending<'a, Vec>, K> { let (mut nodes, key) = subtree.into_parts(); - nodes.sort_by_key(|n| n.j); - // A read returns each entry once. Two rows at one ordinal are the - // server repeating an entry, which would let it line up one container - // as many times as it likes: an honest database never returns one, so - // it is the wrong entry, not a shape the target refuses. - if nodes.windows(2).any(|pair| pair[0].j == pair[1].j) { - return Pending::failed(cipher, wrong("rows of distinct entries")); - } let Some(first) = nodes.first() else { return Pending::ready(cipher, Ok(Vec::new())); }; let header = first.header.clone(); // A read is of one row, so its rows carry one header. Rows of two - // documents are not the answer to it. + // documents are not the answer to it. Checked first: two documents' + // entries can share a slot. if nodes.iter().any(|n| n.header != header) { return Pending::failed(cipher, wrong("rows of one document")); } + // A read returns each entry once. Two rows at one slot are the server + // repeating an entry, which would let it line up one container as many + // times as it likes: an honest database never returns one, so it is the + // wrong entry, not a shape the target refuses. + nodes.sort_by_key(|n| n.j); + if nodes.windows(2).any(|pair| pair[0].j == pair[1].j) { + return Pending::failed(cipher, wrong("rows of distinct entries")); + } let (request, aad) = match retrieval(&header, context) { Ok(parts) => parts, Err(error) => return Pending::failed(cipher, error), @@ -1507,15 +1508,17 @@ enum Under<'p> { /// ([`Error::Aead`]). /// /// The checks the client still makes, each [`Error::JsonWrongEntry`]: no -/// row is placed twice; every row the server returned is one the reads -/// reach; the depth the server reported for a row is the one the chain -/// gives it; a node's selector is its path's, and its decrypted label is the -/// last step of that path; a child's self tag is the one its label's rule -/// derives; the sealed root sits nowhere but at `$`; and a container has the -/// number of children its plaintext says. Under the cascade most of these -/// hold whenever every node opens; they are kept because the node at the -/// top of a read is checked by nothing above it, and because each is -/// cheaper than the AEAD it backs. +/// two rows are one entry ([`open_subtree`], before any key is retrieved); +/// every row the server returned is one the reads reach; the depth the +/// server reported for a row is the one the chain gives it; a child's self +/// tag fits the member or the element rule, and the rule its decrypted +/// label names; the node at the top of a read carries its path's selector +/// and its decrypted label is the last step of that path; the sealed root +/// sits nowhere but at `$`; and a container has the number of children its +/// plaintext says. The node at the top of a read is checked by these alone, +/// since nothing above it is read. Beneath it the cascade already holds +/// every child to its place, and the selector and child-count checks there +/// back the AEAD rather than replace it. /// /// Still allowed by design (ADR 0010): an entry from another row of the /// column, returned with that row's header; a node at the same selector @@ -1633,10 +1636,11 @@ impl<'k> Placing<'k> { d: u32, ) -> Result<(Protected, Label), Error> { let refused = || wrong("a node the walked path reaches"); - // An honest walk reaches each row once. - if std::mem::replace(&mut self.used[at], true) { - return Err(wrong("rows of distinct entries")); - } + // No row is placed twice: below a node the chain follows only + // children that open under its digest, which are the ones sealed + // there, and a key's paths are disjoint, so no row is both a read's + // node and beneath another's. + self.used[at] = true; if self.rows[at].d.is_some_and(|got| got != d) { return Err(wrong("a row at the depth the chain gives it")); } @@ -2922,6 +2926,100 @@ mod tests { } } + /// Seal `value` with its root open, under a fixed key, after `forge` + /// has had the walked nodes' plaintexts, and open it whole: what a + /// sealer that wrote a node's plaintext wrongly would leave behind. + fn seal_forged_and_open( + value: &Value, + forge: impl FnOnce(&mut Vec>>), + ) -> Result { + let context = Context::from_encoded(b"users/profile"); + let mut walked = walk(value, &open_root(), &prf(), &context).unwrap(); + forge(&mut walked.plain); + let key = stack_kms::DataKeyWithTag { + key: stack_kms::DataKey { + iv: [1; 16], + key: [2; 32], + }, + tag: vec![3; 8], + decryption_policy: None, + }; + let aes = leaf_cipher(&key.key).unwrap(); + let doc = seal( + walked, + None, + Uuid::from_u128(1), + key, + &context, + false, + SafeRand::from_controlled_seed(Protected::new([5; 32])), + ) + .unwrap(); + open_whole_chain(&doc, &aes, &context, Selectors::for_tests()) + } + + #[test] + fn a_document_opens_whole_from_its_chain() { + let value = json!({"a": [1, {"b": null}], "c": "d"}); + assert_eq!(seal_forged_and_open(&value, |_| {}).unwrap(), value); + } + + /// A container whose plaintext claims more children than it was sealed + /// over opens (its AAD covers the children the chain found) and is + /// refused by its child count; no honest sealing writes one. + #[test] + fn a_container_claiming_another_child_count_is_refused() { + let value = json!({"a": [1, 2]}); + let refused = seal_forged_and_open(&value, |plain| { + // `$.a`, the second node walked: kind, then the child count. + plain[1] = encode_node(Kind::Array, 3, LabelRef::Key("a"), &json!([])); + }); + assert!( + matches!( + refused, + Err(Error::JsonWrongEntry { + expected: "a container with the children it was sealed over" + }) + ), + "{refused:?}" + ); + } + + /// The whole document opened under another column's PRF context finds + /// no entry carrying the root's token, and the cipher refuses its first + /// entry under the column's AAD. + #[test] + fn a_document_of_another_column_is_refused_by_the_cipher() { + let value = json!({"a": 1}); + let context = Context::from_encoded(b"users/profile"); + let walked = walk(&value, &open_root(), &prf(), &context).unwrap(); + let key = stack_kms::DataKeyWithTag { + key: stack_kms::DataKey { + iv: [1; 16], + key: [2; 32], + }, + tag: vec![3; 8], + decryption_policy: None, + }; + let aes = leaf_cipher(&key.key).unwrap(); + let doc = seal( + walked, + None, + Uuid::from_u128(1), + key, + &context, + false, + SafeRand::from_controlled_seed(Protected::new([5; 32])), + ) + .unwrap(); + let other = Context::from_encoded(b"users/bio"); + let selectors = Selectors::new(Arc::clone(&Selectors::for_tests().keyset), other.clone()); + assert!(matches!( + open_whole_chain(&doc, &aes, &other, selectors), + Err(Error::Aead) + )); + } + #[test] fn too_deep_a_document_is_refused() { // The innermost scalar of `n` nested arrays sits at depth `n`. diff --git a/packages/stack-encrypt/tests/fixtures/json_tree_vectors.json b/packages/stack-encrypt/tests/fixtures/json_tree_vectors.json index 2d14c339b..38207bf2f 100644 --- a/packages/stack-encrypt/tests/fixtures/json_tree_vectors.json +++ b/packages/stack-encrypt/tests/fixtures/json_tree_vectors.json @@ -1,5 +1,5 @@ { - "_comment": "The JSON index's s, x and p per entry, and the tokens a read sends, under the deterministic key source. Read by tests/json_tree_vectors.rs; regenerate with STACK_ENCRYPT_UPDATE_JSON_TREE_VECTORS=1 only for an intended break. Do not edit by hand.", + "_comment": "The JSON index's s, x and p per entry in the stored order, the path stored at each slot (the permutation the seeded generator drew), the selector multiset, q as sets, and the tokens a read sends, under the deterministic key source. Read by tests/json_tree_vectors.rs; regenerate with STACK_ENCRYPT_UPDATE_JSON_TREE_VECTORS=1 only for an intended break. Do not edit by hand.", "document": { "a": { "b": [ @@ -18,25 +18,20 @@ "s": "GUG6+MOjeHGk8vJS5CJOxA==", "x": "PelL2/QWv0M=" }, - { - "p": "8OU5UZoL5Hw=", - "s": "8OVeBxReEgPVrjL7qGYAbg==", - "x": "p7/wbsjaDjQ=" - }, { "p": "WN3/UZRAros=", "s": "HlQhfaUoEj5O+JR/j9trVQ==", "x": "W0aJLUxlVdA=" }, { - "p": "ZOSBKJZJ/NY=", - "s": "0AOmEo3aMm7GHQ6VNNv+Vw==", - "x": "EVNlMjX0dyE=" + "p": "sIWG0EdV0i8=", + "s": "Ie6Xm6ZxkV/9nBoIK7zXsQ==", + "x": "HRuqzbdL4ew=" }, { - "p": "ZHxAvDKdFsI=", - "s": "0AOmEo3aMm7GHQ6VNNv+Vw==", - "x": "q22q1h6JGj0=" + "p": "8OU5UZoL5Hw=", + "s": "8OVeBxReEgPVrjL7qGYAbg==", + "x": "p7/wbsjaDjQ=" }, { "p": "/EJ22V9WvdY=", @@ -44,12 +39,74 @@ "x": "hMhl3B9cj/0=" }, { - "p": "sIWG0EdV0i8=", - "s": "Ie6Xm6ZxkV/9nBoIK7zXsQ==", - "x": "HRuqzbdL4ew=" + "p": "ZHxAvDKdFsI=", + "s": "0AOmEo3aMm7GHQ6VNNv+Vw==", + "x": "q22q1h6JGj0=" + }, + { + "p": "ZOSBKJZJ/NY=", + "s": "0AOmEo3aMm7GHQ6VNNv+Vw==", + "x": "EVNlMjX0dyE=" } ], "iv": "OtEqUO60j5CMut+3HQzYSA==", + "order": [ + "$", + "$.a.b", + "$.d", + "$.a", + "$.a.b[1].c", + "$.a.b[1]", + "$.a.b[0]" + ], + "q": { + "el": [ + { + "el": [], + "sv": [ + { + "s": "0AOmEo3aMm7GHQ6VNNv+Vw==", + "vs": "+nRPy8TKysTmPA8Sps5L7A==" + }, + { + "s": "z1M7JTZ8C8WMd3dqhuZpfA==", + "vs": "u8/P1+1pACZNLu38Mb428w==" + } + ] + } + ], + "sv": [ + { + "s": "0AOmEo3aMm7GHQ6VNNv+Vw==", + "vs": "j5AWPTfxNckQJGx1Zw2b8A==" + }, + { + "s": "8OVeBxReEgPVrjL7qGYAbg==", + "vs": "5SuuFkMUhXimWdtri7FSZg==" + }, + { + "s": "GUG6+MOjeHGk8vJS5CJOxA==", + "vs": "MUinEHjmPaxhTO0p+J6gNw==" + }, + { + "s": "HlQhfaUoEj5O+JR/j9trVQ==", + "vs": "N6jW503hjTBk2DX2bVAvxw==" + }, + { + "s": "Ie6Xm6ZxkV/9nBoIK7zXsQ==", + "vs": "4WXqbvyXIzBV/wIWdYrpCw==" + } + ] + }, + "selectors": [ + "0AOmEo3aMm7GHQ6VNNv+Vw==", + "0AOmEo3aMm7GHQ6VNNv+Vw==", + "8OVeBxReEgPVrjL7qGYAbg==", + "GUG6+MOjeHGk8vJS5CJOxA==", + "HlQhfaUoEj5O+JR/j9trVQ==", + "Ie6Xm6ZxkV/9nBoIK7zXsQ==", + "z1M7JTZ8C8WMd3dqhuZpfA==" + ], "tokens": { "$": [ "tJdy239uYbuNLuRBLfJuNQ==" @@ -78,25 +135,20 @@ "s": "GUG6+MOjeHGk8vJS5CJOxA==", "x": "PelL2/QWv0M=" }, - { - "p": null, - "s": "8OVeBxReEgPVrjL7qGYAbg==", - "x": "wl8ZJzvRwJ0=" - }, { "p": "Dy1WgDveXQU=", "s": "HlQhfaUoEj5O+JR/j9trVQ==", "x": "12lMeBeDMpo=" }, { - "p": "sJXI7nmR44s=", - "s": "0AOmEo3aMm7GHQ6VNNv+Vw==", - "x": "/yUSe3uTE98=" + "p": null, + "s": "Ie6Xm6ZxkV/9nBoIK7zXsQ==", + "x": "uGnxy4JLUhk=" }, { - "p": "d0RUZY7M9PI=", - "s": "0AOmEo3aMm7GHQ6VNNv+Vw==", - "x": "7n2djhOoEXo=" + "p": null, + "s": "8OVeBxReEgPVrjL7qGYAbg==", + "x": "wl8ZJzvRwJ0=" }, { "p": "4mV5CHfjZEg=", @@ -104,12 +156,74 @@ "x": "A/QZeMRJngk=" }, { - "p": null, - "s": "Ie6Xm6ZxkV/9nBoIK7zXsQ==", - "x": "uGnxy4JLUhk=" + "p": "d0RUZY7M9PI=", + "s": "0AOmEo3aMm7GHQ6VNNv+Vw==", + "x": "7n2djhOoEXo=" + }, + { + "p": "sJXI7nmR44s=", + "s": "0AOmEo3aMm7GHQ6VNNv+Vw==", + "x": "/yUSe3uTE98=" } ], "iv": "OtEqUO60j5CMut+3HQzYSA==", + "order": [ + "$", + "$.a.b", + "$.d", + "$.a", + "$.a.b[1].c", + "$.a.b[1]", + "$.a.b[0]" + ], + "q": { + "el": [ + { + "el": [], + "sv": [ + { + "s": "0AOmEo3aMm7GHQ6VNNv+Vw==", + "vs": "+nRPy8TKysTmPA8Sps5L7A==" + }, + { + "s": "z1M7JTZ8C8WMd3dqhuZpfA==", + "vs": "u8/P1+1pACZNLu38Mb428w==" + } + ] + } + ], + "sv": [ + { + "s": "0AOmEo3aMm7GHQ6VNNv+Vw==", + "vs": "j5AWPTfxNckQJGx1Zw2b8A==" + }, + { + "s": "8OVeBxReEgPVrjL7qGYAbg==", + "vs": "5SuuFkMUhXimWdtri7FSZg==" + }, + { + "s": "GUG6+MOjeHGk8vJS5CJOxA==", + "vs": "MUinEHjmPaxhTO0p+J6gNw==" + }, + { + "s": "HlQhfaUoEj5O+JR/j9trVQ==", + "vs": "N6jW503hjTBk2DX2bVAvxw==" + }, + { + "s": "Ie6Xm6ZxkV/9nBoIK7zXsQ==", + "vs": "4WXqbvyXIzBV/wIWdYrpCw==" + } + ] + }, + "selectors": [ + "0AOmEo3aMm7GHQ6VNNv+Vw==", + "0AOmEo3aMm7GHQ6VNNv+Vw==", + "8OVeBxReEgPVrjL7qGYAbg==", + "GUG6+MOjeHGk8vJS5CJOxA==", + "HlQhfaUoEj5O+JR/j9trVQ==", + "Ie6Xm6ZxkV/9nBoIK7zXsQ==", + "z1M7JTZ8C8WMd3dqhuZpfA==" + ], "tokens": { "$": [ "tJdy239uYbuNLuRBLfJuNQ==" diff --git a/packages/stack-encrypt/tests/json_index.rs b/packages/stack-encrypt/tests/json_index.rs index 0ca1634d9..ec3b5d3b9 100644 --- a/packages/stack-encrypt/tests/json_index.rs +++ b/packages/stack-encrypt/tests/json_index.rs @@ -97,10 +97,10 @@ fn ordering(paths: &[&str]) -> JsonOptions { JsonOptions::default().ordering(paths).unwrap() } -/// The design's fixture document. Its entries, by ordinal: `$`, `$.hello`, -/// `$.nested`, `$.nested.deep`, `$.nested.list`, `[0]`, `[0].x`, `[1]`, -/// `[1].x`, `$.number`. The tests pin those ordinals, so the document is -/// fixed rather than generated. +/// The design's fixture document: `$`, `$.hello`, `$.nested`, +/// `$.nested.deep`, `$.nested.list`, `[0]`, `[0].x`, `[1]`, `[1].x`, +/// `$.number`. Its entries are stored in a random order (the root first), +/// so a test finds a node's entry by its path ([`at`]). fn fixture() -> Value { json!({"hello": "world-1", "nested": {"deep": "constant", "list": [{"x": 1}, {"x": 2}]}, "number": 1}) } @@ -191,6 +191,45 @@ async fn keyed(cipher: &Cipher, path: &str, rows: Vec) -> JsonSubtree .open(rows) } +/// The slot the node at the concrete `path` is stored at, found as the +/// server finds it: by the self tag of the path's token, under the root +/// option of `options`. +async fn at_with(cipher: &Cipher, doc: &JsonDocument, path: &str, options: JsonOptions) -> usize { + let key = subtree_key(cipher, path, options).await.unwrap(); + doc.find(&key.token_bytes()[0]) + .unwrap_or_else(|| panic!("the document has {path}")) + .0 +} + +/// [`at_with`] under the sealed root. +async fn at(cipher: &Cipher, doc: &JsonDocument, path: &str) -> usize { + at_with(cipher, doc, path, JsonOptions::default()).await +} + +/// The slots of `paths`, sorted: which entries a set of nodes is stored at. +async fn slots_of(cipher: &Cipher, doc: &JsonDocument, paths: &[&str]) -> Vec { + let mut slots = Vec::with_capacity(paths.len()); + for path in paths { + slots.push(at(cipher, doc, path).await); + } + slots.sort_unstable(); + slots +} + +/// A projection as a set: every group's pairs and element groups sorted, so +/// two projections that hold the same pairs in the same groups compare +/// equal whatever their stored order (`jsonb @>` reads them as sets). +fn as_set(q: &Value) -> Value { + let mut sv = q["sv"].as_array().cloned().unwrap_or_default(); + sv.sort_by_key(|pair| pair.to_string()); + let mut el: Vec = q["el"] + .as_array() + .map(|groups| groups.iter().map(as_set).collect()) + .unwrap_or_default(); + el.sort_by_key(|group| group.to_string()); + json!({"sv": sv, "el": el}) +} + /// Padded standard base64, as the SQL reads and writes it. fn base64(bytes: &[u8]) -> String { use base64ct::{Base64, Encoding}; @@ -393,15 +432,16 @@ mod given_the_fixture_document { assert_eq!(keys, ["h", "q", "sv"], "the document's wire keys"); // `q` is the document's own projection, written once: the needle a // `Contains` of the whole document derives has the same pairs in the - // same groups, and nothing else. + // same groups, and nothing else. Stored, its lists are shuffled; as + // sets, which is how `jsonb @>` reads them, the two are one. let needle = cipher .query(op::Contains, &fixture()) .on(&plan) .await .unwrap(); assert_eq!( - wire["q"], - serde_json::to_value(&needle).unwrap(), + as_set(&wire["q"]), + as_set(&serde_json::to_value(&needle).unwrap()), "`q` is the needle a `Contains` of the whole document derives" ); assert!( @@ -456,19 +496,32 @@ mod given_the_fixture_document { assert_eq!(entries.len(), 10, "one entry per node of the fixture"); assert!( entries[0].parent_tag().is_none(), - "the root has no parent tag" + "the root, at slot 0, has no parent tag" ); - for head in [&entries[1], &entries[2], &entries[9]] { + let heads = slots_of(&cipher, &doc, &["$.hello", "$.nested", "$.number"]).await; + for j in &heads { assert!( - head.parent_tag().is_none(), + entries[*j].parent_tag().is_none(), "a head has no parent tag under the sealed root" ); } - for (j, below) in entries[3..9].iter().enumerate() { + let below = slots_of( + &cipher, + &doc, + &[ + "$.nested.deep", + "$.nested.list", + "$.nested.list[0]", + "$.nested.list[0].x", + "$.nested.list[1]", + "$.nested.list[1].x", + ], + ) + .await; + for j in &below { assert!( - below.parent_tag().is_some(), - "entry {} below a head carries a parent tag", - j + 3 + entries[*j].parent_tag().is_some(), + "entry {j} below a head carries a parent tag" ); } // Every entry carries a self tag, the root and the heads included, and @@ -480,9 +533,9 @@ mod given_the_fixture_document { "one distinct self tag per entry" ); // Every parent tag is distinct: siblings are not linkable at rest. - let tags: std::collections::HashSet<_> = entries[3..9] + let tags: std::collections::HashSet<_> = below .iter() - .map(|e| *e.parent_tag().unwrap()) + .map(|j| *entries[*j].parent_tag().unwrap()) .collect(); assert_eq!(tags.len(), 6, "every parent tag is distinct"); // The projection groups the two list elements: each is a nested group @@ -514,19 +567,29 @@ mod given_the_fixture_document { #[tokio::test] async fn gives_equal_values_at_one_path_one_value_selector_and_two_ciphertexts() { let Scenario { cipher, .. } = scenario().await; - let pair = |doc: &JsonDocument, i: usize| -> (String, String) { + // The value selectors of the elements: the pairs of `q` at the + // elements' path, in their stored (shuffled) order. + let element_vs = |doc: &JsonDocument| -> Vec { + let s = base64(doc.entries()[1].selector()); let q = serde_json::to_value(doc.projection()).unwrap(); - ( - q["sv"][i]["s"].as_str().unwrap().to_owned(), - q["sv"][i]["vs"].as_str().unwrap().to_owned(), - ) + q["sv"] + .as_array() + .unwrap() + .iter() + .filter(|pair| pair["s"] == json!(s)) + .map(|pair| pair["vs"].as_str().unwrap().to_owned()) + .collect() }; let same = seal(&cipher, &json!([1, 1])).await; - let (a, b) = (&same.entries()[1], &same.entries()[2]); + let (a, b) = ( + &same.entries()[at(&cipher, &same, "$[0]").await], + &same.entries()[at(&cipher, &same, "$[1]").await], + ); assert_eq!(a.selector(), b.selector(), "two elements share a path"); + let same_vs = element_vs(&same); + assert_eq!(same_vs.len(), 2, "one pair per element"); assert_eq!( - pair(&same, 1), - pair(&same, 2), + same_vs[0], same_vs[1], "equal values at one path share a value selector" ); assert_ne!( @@ -535,16 +598,18 @@ mod given_the_fixture_document { "equal values do not share a ciphertext" ); let different = seal(&cipher, &json!([1, 2])).await; - let (one, two) = (&different.entries()[1], &different.entries()[2]); + let (one, two) = ( + &different.entries()[at(&cipher, &different, "$[0]").await], + &different.entries()[at(&cipher, &different, "$[1]").await], + ); assert_eq!(one.selector(), two.selector(), "one path"); + let different_vs = element_vs(&different); assert_ne!( - pair(&different, 1), - pair(&different, 2), + different_vs[0], different_vs[1], "different values at one path have different value selectors" ); - assert_eq!( - pair(&different, 1), - pair(&same, 1), + assert!( + different_vs.contains(&same_vs[0]), "the value selector names the value at its path, in any document" ); } @@ -553,8 +618,9 @@ mod given_the_fixture_document { async fn chains_the_top_level_fields_from_an_open_root() { let Scenario { cipher, .. } = scenario().await; let open_doc = seal_with(&cipher, &fixture(), &open_root()).await; + let hello = at_with(&cipher, &open_doc, "$.hello", open_root()).await; assert!( - open_doc.entries()[1].parent_tag().is_some(), + open_doc.entries()[hello].parent_tag().is_some(), "without the sealed root, the top-level fields chain from the root" ); } @@ -568,16 +634,18 @@ mod given_the_fixture_document { let path = JsonPath::parse(path).unwrap(); keyset.json_selector(&path, nonempty!("users").with("profile")) }; - for (path, j) in [ - ("$.nested.deep", 3), - ("$.nested.list[*].x", 6), - ("$.nested.list[1].x", 8), - ("$", 0), + for (path, node) in [ + ("$.nested.deep", "$.nested.deep"), + ("$.nested.list[*].x", "$.nested.list[0].x"), + ("$.nested.list[1].x", "$.nested.list[1].x"), + ("$.nested.list[1].x", "$.nested.list[0].x"), + ("$", "$"), ] { + let j = at(&cipher, &doc, node).await; assert_eq!( selector(path).await.unwrap().as_bytes(), doc.entries()[j].selector(), - "{path} derives entry {j}'s stored selector" + "{path} derives {node}'s stored selector" ); } assert_ne!( @@ -657,9 +725,22 @@ mod given_ordered_paths { async fn puts_a_term_only_on_a_number_or_string_leaf_at_an_ordered_path() { let Scenario { cipher, .. } = scenario().await; let doc = seal_with(&cipher, &fixture(), &ordering(&["$.hello", "$.number"])).await; - assert_eq!(with_term(&doc), [1, 9], "$.hello and $.number"); + assert_eq!( + with_term(&doc), + slots_of(&cipher, &doc, &["$.hello", "$.number"]).await, + "$.hello and $.number" + ); let wild = seal_with(&cipher, &fixture(), &ordering(&["$.nested.list[*].x"])).await; - assert_eq!(with_term(&wild), [6, 8], "a wildcard covers every element"); + assert_eq!( + with_term(&wild), + slots_of( + &cipher, + &wild, + &["$.nested.list[0].x", "$.nested.list[1].x"] + ) + .await, + "a wildcard covers every element" + ); // A container, a boolean or a null at an ordered path carries none. let kinds = seal_with( &cipher, @@ -667,8 +748,11 @@ mod given_ordered_paths { &ordering(&["$.a", "$.b", "$.c", "$.e", "$.e[*]"]), ) .await; - // $, $.a, $.b, $.c, $.c.d, $.e, $.e[0], $.e[1] - assert_eq!(with_term(&kinds), [6, 7], "only the number and the string"); + assert_eq!( + with_term(&kinds), + slots_of(&cipher, &kinds, &["$.e[0]", "$.e[1]"]).await, + "only the number and the string" + ); let scalar = seal_with(&cipher, &json!(7), &ordering(&["$"])).await; assert_eq!(with_term(&scalar), [0], "the root scalar itself, at `$`"); assertions::opens_to(&cipher, doc, &fixture(), "ordered at $.hello and $.number").await; @@ -732,17 +816,29 @@ mod given_a_subtree_key { "the key carries the path it was derived from" ); let under = doc.subtree(&key.token_bytes()); - let found: Vec = under.iter().map(|(j, _, _)| *j).collect(); + let mut expected = Vec::new(); + for (path, d) in [ + ("$.nested", 0), + ("$.nested.deep", 1), + ("$.nested.list", 1), + ("$.nested.list[0]", 2), + ("$.nested.list[0].x", 3), + ("$.nested.list[1]", 2), + ("$.nested.list[1].x", 3), + ] { + expected.push((at(&cipher, &doc, path).await, d)); + } assert_eq!( - found, - [2, 3, 4, 5, 6, 7, 8], - "nested itself, then deep, list, both elements, both leaves" + under.first().map(|(j, _, d)| (*j, *d)), + Some(expected[0]), + "nested itself first" ); - let depths: Vec = under.iter().map(|(_, _, d)| *d).collect(); + let mut found: Vec<(usize, u32)> = under.iter().map(|(j, _, d)| (*j, *d)).collect(); + found.sort_unstable(); + expected.sort_unstable(); assert_eq!( - depths, - [0, 1, 1, 2, 3, 2, 3], - "each row's depth below $.nested" + found, expected, + "nested, deep, list, both elements and both leaves, each at its depth below $.nested" ); assert!( doc.subtree(&[[7; 16]]).is_empty(), @@ -768,16 +864,20 @@ mod given_a_subtree_key { assert_ne!(deeper.token_bytes(), key.token_bytes(), "not nested's"); assert_eq!( subtree_ordinals(&doc, &deeper.token_bytes()), - [7, 8], + [ + at(&cipher, &doc, "$.nested.list[1]").await, + at(&cipher, &doc, "$.nested.list[1].x").await + ], "the second element and its x" ); + let list_at = at(&cipher, &doc, "$.nested.list").await; let list = subtree_key(&cipher, "$.nested.list", JsonOptions::default()) .await .unwrap(); let (_, entry, list_token) = doc .children(&key.token_bytes()[0]) .into_iter() - .find(|(j, _, _)| *j == 4) + .find(|(j, _, _)| *j == list_at) .expect("the list is a child of nested"); assert_eq!( list_token, @@ -786,10 +886,10 @@ mod given_a_subtree_key { ); assert_eq!( doc.find(&list.token_bytes()[0]).map(|(j, _)| j), - Some(4), + Some(list_at), "the list's token finds the list" ); - assert_eq!(entry.selector(), doc.entries()[4].selector()); + assert_eq!(entry.selector(), doc.entries()[list_at].selector()); let children: Vec = doc .children(&list.token_bytes()[0]) .iter() @@ -797,7 +897,10 @@ mod given_a_subtree_key { .collect(); assert_eq!( children, - [5, 7], + [ + at(&cipher, &doc, "$.nested.list[0]").await, + at(&cipher, &doc, "$.nested.list[1]").await + ], "the children of a node, by ordinal: the list's two elements" ); assert!(doc.children(&[7; 16]).is_empty(), "no node, no children"); @@ -833,8 +936,11 @@ mod given_a_subtree_key { let second_row = subtree_key(&cipher, "$[1]", open_root()).await.unwrap(); assert_eq!( subtree_ordinals(&array_doc, &second_row.token_bytes()), - [4, 5], - "$, [0], [0][0], [0][1], [1], [1][0]" + [ + at_with(&cipher, &array_doc, "$[1]", open_root()).await, + at_with(&cipher, &array_doc, "$[1][0]", open_root()).await + ], + "the second row and its element" ); } } @@ -858,12 +964,14 @@ mod given_a_position { .unwrap(); let array = seal(&cipher, &json!({"a": [{"x": 2}], "h": {"a": [{"x": 2}]}})).await; let found = subtree_ordinals(&array, &first.token_bytes()); - let at = array - .entries() - .iter() - .position(|e| e.self_tag() == array.entries()[found[0]].self_tag()) - .unwrap(); - assert_eq!(found, [at, at + 1], "{path}: the element and its x"); + assert_eq!( + found, + [ + at(&cipher, &array, path).await, + at(&cipher, &array, &format!("{path}.x")).await + ], + "{path}: the element and its x" + ); for object in [ json!({"a": {"k": {"x": 1}}, "h": {"a": {"k": {"x": 1}}}}), json!({"a": {"k": [1, 2]}, "h": {"a": {"k": [1, 2]}}}), @@ -890,35 +998,34 @@ mod given_an_extracted_node { async fn opens_to_its_scalar_and_a_walked_subtree_to_its_value() { let Scenario { cipher, .. } = scenario().await; let doc = seal(&cipher, &fixture()).await; - let node = |j: usize| doc.node(j).unwrap(); - let scalar = |path: &'static str, j: usize| { + let scalar = |path: &'static str| { let cipher = &cipher; - let node = node(j); + let doc = &doc; async move { + let node = doc.node(at(cipher, doc, path).await).unwrap(); cipher .decrypt_as::(extracted(cipher, path, node).await, aead()) .await } }; assert_eq!( - scalar("$.hello", 1).await.unwrap(), + scalar("$.hello").await.unwrap(), json!("world-1"), "$.hello opens to its scalar" ); assert_eq!( - scalar("$.nested.list[0].x", 6).await.unwrap(), + scalar("$.nested.list[0].x").await.unwrap(), json!(1), "[0].x opens to its scalar" ); assert_eq!( - scalar("$", 0).await.unwrap(), + scalar("$").await.unwrap(), fixture(), "the sealed root is the document" ); - assertions::refused_as_opened_type( - &scalar("$.nested", 2).await, - None, - "a container node holds no value of its own", + assertions::refused_as_tampered( + &scalar("$.nested").await, + "a container is sealed over its children's tags and does not open alone", ); let key = subtree_key(&cipher, "$.nested", JsonOptions::default()) @@ -966,11 +1073,12 @@ mod given_an_extracted_node { async fn round_trips_with_the_envelope_the_sql_grafts() { let Scenario { cipher, .. } = scenario().await; let doc = seal(&cipher, &fixture()).await; - let mut wire = serde_json::to_value(doc.node(6).unwrap()).unwrap(); + let x = at(&cipher, &doc, "$.nested.list[0].x").await; + let mut wire = serde_json::to_value(doc.node(x).unwrap()).unwrap(); wire["i"] = json!({"t": "users", "c": "profile"}); wire["v"] = json!(4); let back: JsonNode = serde_json::from_value(wire).unwrap(); - assert_eq!(back.ordinal(), 6, "the node keeps its ordinal"); + assert_eq!(back.ordinal(), x as u64, "the node keeps its slot"); assert_eq!( cipher .decrypt_as::( @@ -1109,7 +1217,8 @@ mod given_a_derived_record { async fn opens_each_stored_form_as_its_field() { let Scenario { cipher, .. } = scenario().await; let doc = seal(&cipher, &fixture()).await; - let node = extracted(&cipher, "$.hello", doc.node(1).unwrap()).await; + let hello = at(&cipher, &doc, "$.hello").await; + let node = extracted(&cipher, "$.hello", doc.node(hello).unwrap()).await; let list = subtree_key(&cipher, "$.nested.list", JsonOptions::default()) .await .unwrap(); @@ -1341,10 +1450,11 @@ mod given_rows_the_server_chose { use super::*; /// The rows of a walk come from the server, which can repeat an entry or - /// pick which entries to return. Every entry is authentic, but the - /// rebuild nests the containers it is handed: a repeated row is refused, - /// and so is a line of containers that are not one subtree's consecutive - /// entries, before the rebuild can recurse as deep as the rows. + /// pick which entries to return. Every entry is authentic: a repeated + /// row is refused before any key is retrieved, and a line of containers + /// that are not one subtree is refused by the cascade at the first of + /// them, whose ciphertext commits to children the rows do not hold, so + /// the open never recurses past a node that did not open. #[tokio::test] async fn refuses_a_repeated_row_or_a_line_of_containers() { let Scenario { cipher, .. } = scenario().await; @@ -1367,13 +1477,16 @@ mod given_rows_the_server_chose { .collect(), ); let doc = seal(&cipher, &wide).await; - let last = doc.entries().len() - 1; - let containers: Vec = (1..doc.entries().len()) - .filter(|j| j % 2 == 1 || *j == last) - .map(|j| doc.node(j).unwrap()) - .collect(); + let mut containers = Vec::new(); + for i in 0..200 { + containers.push( + doc.node(at(&cipher, &doc, &format!("$.k{i:03}")).await) + .unwrap(), + ); + } + containers.push(doc.node(at(&cipher, &doc, "$.k199.x").await).unwrap()); assert_eq!(containers.len(), 201, "200 objects and the last leaf"); - assertions::refused_as_wrong_entry( + assertions::refused_as_tampered( &cipher .decrypt_as::, JsonSubtree>( keyed(&cipher, "$.k000", containers).await, @@ -1391,17 +1504,20 @@ mod given_rows_the_server_chose { .collect(), ); let doc = seal(&cipher, &wide).await; - let last = doc.entries().len() - 1; - let arrays: Vec = (1..doc.entries().len()) - .filter(|j| *j == 1 || j % 3 == 2 || *j == last) - .map(|j| doc.node(j).unwrap()) - .collect(); + let mut arrays = vec![doc.node(at(&cipher, &doc, "$.k000").await).unwrap()]; + for i in 0..200 { + arrays.push( + doc.node(at(&cipher, &doc, &format!("$.k{i:03}[0]")).await) + .unwrap(), + ); + } + arrays.push(doc.node(at(&cipher, &doc, "$.k199[0][0]").await).unwrap()); assert_eq!( arrays.len(), 202, "the outer array, 200 inner and the last leaf" ); - assertions::refused_as_wrong_entry( + assertions::refused_as_tampered( &cipher .decrypt_as::, JsonSubtree>( keyed(&cipher, "$.k000", arrays).await, @@ -1439,10 +1555,11 @@ mod given_a_tampered_document { "regrafted from another document", ); + // The root's ciphertext commits to both members' tags. let truncated = tamper(wire.clone(), &|sv| { let _ = sv.pop(); }); - assertions::refused_as_opened_type(&open(&cipher, truncated).await, None, "truncated"); + assertions::refused_as_tampered(&open(&cipher, truncated).await, "truncated"); let flipped = tamper(wire.clone(), &|sv| { let c = sv[1]["c"].as_str().unwrap().to_owned(); @@ -1460,16 +1577,17 @@ mod given_a_tampered_document { .await; assertions::refused_as_tampered(&wrong_context, "another column"); - // A node lifted out of its document fails under another ordinal. + // A node lifted out of its document fails under another slot. let sealed = seal(&cipher, &value).await; - let mut lifted = serde_json::to_value(sealed.node(1).unwrap()).unwrap(); - lifted["j"] = json!(2); + let a = at(&cipher, &sealed, "$.a").await; + let mut lifted = serde_json::to_value(sealed.node(a).unwrap()).unwrap(); + lifted["j"] = json!(3 - a); let lifted: JsonNode = serde_json::from_value(lifted).unwrap(); assertions::refused_as_tampered( &cipher .decrypt_as::(extracted(&cipher, "$.a", lifted).await, aead()) .await, - "a node under another ordinal", + "a node under another slot", ); } } @@ -2158,7 +2276,7 @@ mod given_a_float_at_full_precision { .await .unwrap(); let rows = read_rows(&sealed, &key); - let element = sealed.node(2).unwrap(); + let element = sealed.node(at(cipher, &sealed, "$.list[0]").await).unwrap(); vec![ ( "the document, from the sealed root", @@ -2371,7 +2489,8 @@ mod given_an_accessor { let Scenario { cipher, .. } = scenario().await; let plan = ordered_plan(); let doc = cipher.encrypt(&fixture()).using(&plan).await.unwrap(); - let number = &doc.entries()[9]; + let number = &doc.entries()[at(&cipher, &doc, "$.number").await]; + let hello = &doc.entries()[at(&cipher, &doc, "$.hello").await]; let bound = cipher .query(op::AtPath(op::Gte), &("$.number", 0)) .on(&plan) @@ -2401,7 +2520,7 @@ mod given_an_accessor { .await .unwrap(); assert!( - text.ordering_term() < doc.entries()[1].ordering_term().unwrap(), + text.ordering_term() < hello.ordering_term().unwrap(), "world-0 < world-1" ); let range = cipher diff --git a/packages/stack-encrypt/tests/json_leakage/calibration.rs b/packages/stack-encrypt/tests/json_leakage/calibration.rs index 6def548e1..d33599f6c 100644 --- a/packages/stack-encrypt/tests/json_leakage/calibration.rs +++ b/packages/stack-encrypt/tests/json_leakage/calibration.rs @@ -15,10 +15,10 @@ use super::distinguishers::{self, walk_positions}; use super::mutants; use super::observe::{ObservedQuery, ObservedWorld}; use super::profile::{ - flatten, leak, Clauses, Column, KeyOrder, KindOrder, Leakage, NumberOrder, OrderKind, - Placement, Query, QueryLeakage, StringOrder, + leak, Clauses, Column, KeyOrder, KindOrder, Leakage, NumberOrder, OrderKind, Placement, Query, + QueryLeakage, StringOrder, }; -use super::{real, simulator}; +use super::{canonical, real, simulator}; use crate::common::stack_cipher; /// The seed of every simulator run and every mutant's choices. Any seed @@ -70,11 +70,14 @@ fn placements(leakage: &Leakage) -> Vec> { } } -/// Every distinguisher's feature of `world`, by name. +/// Every distinguisher's feature of `world`, by name, read in its +/// canonical order ([`canonical`]): the stored order is random in the real +/// scheme and in the simulation, independently. fn features(world: &ObservedWorld) -> Vec<(&'static str, String)> { + let world = canonical::canonical(world); distinguishers::all() .into_iter() - .map(|d| (d.name, (d.run)(world))) + .map(|d| (d.name, (d.run)(&world))) .collect() } @@ -304,14 +307,15 @@ mod given_integers_ordered_exactly { mod given_the_jsonb_reading_of_the_key_order { use super::*; - /// The page's "sorted order of their key names" does not say which - /// sort. Read as `jsonb`'s (shorter keys first), the profile is told - /// from the real scheme on a row whose keys sort differently the two - /// ways (`{"a": …, "a b": …, "é": …, "0": …}`): the sort is the keys' - /// UTF-8 bytes, and a reader inferring key names from the order needs to - /// know that. + /// The stored order no longer shows the key order, but a containment + /// needle is sent in the walk's order, an object's members sorted by + /// their keys' UTF-8 bytes. Read as `jsonb`'s sort (shorter keys first), + /// the profile is told from the real scheme by a needle whose keys sort + /// differently the two ways (`{"a": …, "a b": …, "é": …, "0": …}`): a + /// needle with sibling keys tells the database their byte order, and a + /// reader inferring key names from it needs to know which sort it is. #[test] - fn the_stored_order_tells_it_apart() { + fn a_needle_tells_it_apart() { let runtime = runtime(); let cipher = runtime.block_on(stack_cipher()); let clauses = Clauses { @@ -326,8 +330,8 @@ mod given_the_jsonb_reading_of_the_key_order { .map(|(n, _, _)| n) .collect(); assert!( - fired.contains(&"selector_pattern_across_rows"), - "the order of a row's selectors tells byte order from jsonb order: {fired:?}" + fired.contains(&"transcripts"), + "the order of a needle's pairs tells byte order from jsonb order: {fired:?}" ); } } @@ -602,45 +606,74 @@ mod the_harness { ); } - /// The page's claim that the stored order reveals each row's shape holds - /// on a column whose rows vary: the order-only attack of - /// `distinguishers::depth_recovery` recovers every row's depth sequence - /// exactly, from the stored rows alone. + /// The stored order says nothing beyond the root's place: sealing one + /// document four times stores its root at slot `0` every time and its + /// other entries, and the pairs of `q`, in a fresh order each time. The + /// document has twenty nodes, each at a path of its own and none inside + /// an array, so its selectors tell each seal's order and `q` is one + /// group of twenty pairs. The real scheme draws each order from the + /// operating system: two seals put the nineteen other entries in one + /// order with probability `1 / 19!`, about `10^-17`, and `q`'s pairs + /// with `1 / 20!`, so across the six pairs of seals this fails by chance + /// below `10^-15`. #[test] - fn the_order_reveals_the_shape() { + fn the_order_is_drawn_afresh_with_the_root_first() { let runtime = runtime(); let cipher = runtime.block_on(stack_cipher()); - let docs = vec![ - json!({"a": {"b": 1, "c": {"d": 2}}, "e": [1, {"f": 3}]}), - json!({"a": {"c": {"d": 2}}, "e": []}), - json!({"a": {"b": 1}, "e": [{"f": 3}]}), - json!({"e": [{"f": 1}, {"g": 2}]}), - json!({"e": [{"g": 1}]}), - json!({"a": {"c": {}}}), - ]; + let object = |prefix: &str, n: usize| { + serde_json::Value::Object((0..n).map(|i| (format!("{prefix}{i}"), json!(i))).collect()) + }; + let doc = json!({"a": object("a", 6), "b": object("b", 6), "c": object("c", 4)}); let world = World { - name: "varied shapes", + name: "one document, sealed four times", column: Column { sealed_root: true, ordered: Vec::new(), }, - docs: docs.clone(), - queries: Vec::new(), + docs: vec![doc; 4], + queries: vec![Query::Extract("$".into())], }; let observed = real_world(&runtime, &cipher, &world); - let truth: Vec>> = docs + let ObservedQuery::Extract { s: root } = observed.queries[0] else { + unreachable!("an extraction"); + }; + let orders: Vec> = observed + .rows .iter() - .map(|doc| { - flatten(doc, KeyOrder::Utf8Bytes) - .iter() - .map(|n| Some(n.positional.len())) - .collect() + .map(|row| row.entries.iter().map(|e| e.s).collect()) + .collect(); + let projections: Vec> = observed + .rows + .iter() + .map(|row| { + assert!(row.q.elements.is_empty(), "no array, one group"); + row.q.pairs.iter().map(|(s, _)| *s).collect() }) .collect(); - assert_eq!( - distinguishers::depth_recovery(&observed), - truth, - "the stored order alone gives every row's depth sequence" - ); + let mut first = orders[0].clone(); + first.sort_unstable(); + for (order, pairs) in orders.iter().zip(&projections) { + assert_eq!(order.len(), 20, "one entry per node"); + assert_eq!(order[0], root, "the root at slot 0"); + let mut sorted = order.clone(); + sorted.sort_unstable(); + assert_eq!(sorted, first, "one document: one multiset of selectors"); + let mut paired = pairs.clone(); + paired.sort_unstable(); + assert_eq!(paired, first, "`q` holds one pair per node"); + } + for i in 0..orders.len() { + for j in i + 1..orders.len() { + assert_ne!( + orders[i][1..], + orders[j][1..], + "seals {i} and {j} store the other entries in one order" + ); + assert_ne!( + projections[i], projections[j], + "seals {i} and {j} store `q`'s pairs in one order" + ); + } + } } } diff --git a/packages/stack-encrypt/tests/json_leakage/canonical.rs b/packages/stack-encrypt/tests/json_leakage/canonical.rs new file mode 100644 index 000000000..b1cb418a0 --- /dev/null +++ b/packages/stack-encrypt/tests/json_leakage/canonical.rs @@ -0,0 +1,637 @@ +//! The canonical order: each stored row's entries and every list of its +//! projection `q`, put in an order an observer computes from the column and +//! the queries without the stored order, so that the distinguishers can +//! name an entry by its place and still compare a real world with a +//! simulated one for equality. +//! +//! The stored order is random ([`super::profile::StoredOrder::Random`]), in +//! the real scheme and in a faithful simulation alike, independently. A +//! feature that named an entry by its slot, or a selector by the slot it +//! first appears at, would tell the two apart by chance alone. So every +//! distinguisher reads the world in this order instead: +//! +//! - Every name (a selector or a value selector) gets a rank from what is +//! attached to it, in tiers compared in turn. At rest: a selector by, per +//! row, whether the root carries it, a description of each entry carrying +//! it, and its `q` pairs by their group's depth and their value +//! selector's colour (per row, the depths it is stored at; per needle, the +//! depths it is sent at); a value selector by that colour. Then, +//! repeatedly until nothing more splits, the groups of `q` a name's pairs +//! sit in, by the ranks of their pairs, of their parent group's and of +//! their element groups'. Then the queries that name it (a needle's pairs +//! by their place in it, since a needle is sent in the walk's order), +//! then where walks reach a selector's entries, then the needles' groups +//! as `q`'s were. Names still tied are ones nothing tells apart (`$.a.b` +//! and `$.a.b.c` in a row `{"a": {"b": {"c": {}}}}`, at rest); they are +//! set apart one at a time, each followed by the refinement again, so +//! that one choice orders the entries and `q` alike. +//! - Each row keeps its root at slot `0`, where every document stores it. +//! The other entries are sorted by their selector's rank, their own +//! description (ciphertext length, length in characters, wire fields, +//! whether they carry a parent and a self tag, and their ordering term's +//! rank among every term in the world), the first query and token whose +//! token names them, and the first place a walk reaches them below the +//! node a token names: the query, the token, and for each node on the way +//! down what the walk shows beneath it and, for an array's elements, the +//! ordinal (an object's members' ordinals are a keyed rank, a random +//! permutation that would order them differently in every world). +//! - Every list of `q` is sorted by its pairs' ranks, and its element +//! groups by their sorted contents. +//! +//! Two entries the sort ties agree on everything a distinguisher reads of +//! them but their random bytes (a ciphertext, a self tag or a parent tag no +//! token links), so which goes first changes no feature, and ties keep +//! their stored order. A tie between two entries some feature does tell +//! apart would make the positive control fail by chance; the descriptions +//! above are what keeps that from happening, and one that leaves out +//! something a feature reads is the first place to look if it does. The +//! walk tiers come after everything at rest, so that a world whose walks +//! differ (a mutant's) reorders only entries that agree at rest, and a +//! named node's place does not move with what the walks below it reach. +//! +//! Needles are sent in the walk's order, which is part of what they reveal +//! ([`super::profile::GroupLeakage::pairs`]), and are left as sent. Each +//! entry's stored slot is kept beside it ([`ObservedRow::slots`]) for the +//! feature that reads the stored order itself +//! ([`super::distinguishers::walked_subtrees_are_contiguous`]). + +use std::collections::{BTreeMap, BTreeSet, HashMap}; + +use super::chain::Tag16; +use super::distinguishers::{row_index, walk_with, Hit}; +use super::observe::{ObservedEntry, ObservedGroup, ObservedQuery, ObservedRow, ObservedWorld}; + +/// A place a walk reaches an entry below the node a token names: the +/// query's place among the queries, the token's place in its list, and a +/// step for each node on the way down to the entry ([`steps`]). +type Address = (usize, usize, Vec); + +/// Where the walks find an entry: the first query and token whose token +/// names it, and the first place a walk reaches it below the node a token +/// names. +type Found = (Option<(usize, usize)>, Option
); + +/// What an entry shows at rest, its selector aside: its ciphertext's length, +/// its length in characters, its wire fields, whether it carries a parent +/// tag and a self tag, and its ordering term's rank. +type EntryKey = (usize, usize, Vec, bool, bool, Option); + +/// A projection and what it is: a row's `q`, or a needle or `alt` sent. +type Projection<'a> = (String, &'a ObservedGroup); + +/// `world` in the canonical order. +pub fn canonical(world: &ObservedWorld) -> ObservedWorld { + let terms = term_ranks(world); + let colours = value_colours(world); + let rest = rest_ranks(world, &terms, &colours); + let walks = walks(world); + let ranks = ranks(world, &terms, &walks, &rest); + let found = addresses(world, &walks, &terms, &ranks); + let rows = world + .rows + .iter() + .zip(&found) + .map(|(row, found)| { + let mut order: Vec = (1..row.entries.len()).collect(); + order.sort_by_key(|&j| { + ( + ranks[&row.entries[j].s], + entry_key(&row.entries[j], &terms), + found[j].clone(), + ) + }); + if !row.entries.is_empty() { + order.insert(0, 0); + } + ObservedRow { + entries: order.iter().map(|&j| row.entries[j].clone()).collect(), + slots: order.iter().map(|&j| row.slots[j]).collect(), + q: group(&row.q, &ranks), + ..row.clone() + } + }) + .collect(); + ObservedWorld { + rows, + queries: world.queries.clone(), + query_fields: world.query_fields.clone(), + } +} + +/// Every ordering term in the world, stored and sent, by its rank in byte +/// order: the order the database compares them in. +fn term_ranks(world: &ObservedWorld) -> HashMap, usize> { + let mut all: Vec<&Vec> = world + .rows + .iter() + .flat_map(|r| r.entries.iter().filter_map(|e| e.op.as_ref())) + .collect(); + all.extend(world.queries.iter().filter_map(|q| match q { + ObservedQuery::Order { op, .. } => Some(op), + _ => None, + })); + all.sort_unstable(); + all.dedup(); + all.into_iter() + .enumerate() + .map(|(rank, op)| (op.clone(), rank)) + .collect() +} + +fn entry_key(entry: &ObservedEntry, terms: &HashMap, usize>) -> EntryKey { + ( + entry.c.len(), + entry.chars, + entry.fields.clone(), + entry.p.is_some(), + entry.x.is_some(), + entry.op.as_ref().map(|op| terms[op]), + ) +} + +/// One walk: the query's place among the queries, the row, the token's +/// place in its list, and what the walk finds, the named node first. +type Walk = (usize, usize, usize, Vec); + +/// Every walk in the world that finds something. +fn walks(world: &ObservedWorld) -> Vec { + let indexes: Vec<_> = world.rows.iter().map(row_index).collect(); + let mut out = Vec::new(); + for (i, query) in world.queries.iter().enumerate() { + let ObservedQuery::Subtree { tokens, .. } = query else { + continue; + }; + for (r, (row, index)) in world.rows.iter().zip(&indexes).enumerate() { + for (k, token) in tokens.iter().enumerate() { + let hits = walk_with(row, index, token); + if !hits.is_empty() { + out.push((i, r, k, hits)); + } + } + } + } + out +} + +/// For every row and every stored slot, where the walks find the entry +/// there, each node on the way described with `ranks`. +fn addresses( + world: &ObservedWorld, + walks: &[Walk], + terms: &HashMap, usize>, + ranks: &HashMap, +) -> Vec> { + let mut out: Vec> = world + .rows + .iter() + .map(|row| vec![(None, None); row.entries.len()]) + .collect(); + for (i, r, k, hits) in walks { + let (i, r, k) = (*i, *r, *k); + let at = &mut out[r][hits[0].position].0; + if at.is_none_or(|before| (i, k) < before) { + *at = Some((i, k)); + } + for (position, path) in steps(&world.rows[r], hits, terms, ranks) { + let here = (i, k, path); + let at = &mut out[r][position].1; + if at.as_ref().is_none_or(|before| here < *before) { + *at = Some(here); + } + } + } + out +} + +/// For every selector, per row, where the walks find each entry carrying +/// it, sorted. +fn walk_tier(world: &ObservedWorld, found: &[Vec]) -> HashMap> { + let mut out: HashMap> = HashMap::new(); + for (r, row) in world.rows.iter().enumerate() { + for (j, entry) in row.entries.iter().enumerate() { + out.entry(entry.s) + .or_default() + .push((r, found[r][j].clone())); + } + } + out.values_mut().for_each(|found| found.sort_unstable()); + out +} + +/// Every hit of one walk below the node it names, by its position, with its +/// path: for each node on the way down, its step under its parent and what +/// the walk shows beneath it (the node's selector's rank, its own +/// description, the rule its self tag matched, and the same of its children, +/// sorted). A step is the node's ordinal where it shares its selector with a +/// sibling (an array's elements, whose ordinal is their index) and nothing +/// where it does not (an object's members, whose ordinals are a keyed rank: a +/// random permutation, which would order the members differently in every +/// world). +fn steps( + row: &ObservedRow, + hits: &[Hit], + terms: &HashMap, usize>, + ranks: &HashMap, +) -> Vec<(usize, Vec)> { + let mut first: HashMap = HashMap::new(); + for (h, hit) in hits.iter().enumerate() { + let _ = first.entry(hit.position).or_insert(h); + } + // The hit each hit hangs off: walks go depth first, so a parent comes + // before its children. + let parent: Vec> = hits + .iter() + .enumerate() + .map(|(h, hit)| { + hit.parent + .and_then(|p| first.get(&p).copied()) + .filter(|&p| p < h) + }) + .collect(); + let mut kids: Vec> = vec![Vec::new(); hits.len()]; + for (h, p) in parent.iter().enumerate() { + if let Some(p) = p { + kids[*p].push(h); + } + } + let s = |h: usize| row.entries[hits[h].position].s; + let step = |h: usize, p: usize| { + let twins = kids[p].iter().filter(|&&d| s(d) == s(h)).count(); + hits[h].ordinal.filter(|_| twins > 1) + }; + let mut below: Vec = vec![String::new(); hits.len()]; + for h in (0..hits.len()).rev() { + let mut children: Vec = kids[h] + .iter() + .map(|&c| format!("{:?}{}", step(c, h), below[c])) + .collect(); + children.sort_unstable(); + below[h] = format!( + "({} {:?} {:?} {children:?})", + ranks[&s(h)], + entry_key(&row.entries[hits[h].position], terms), + hits[h].rule + ); + } + let mut paths: Vec> = vec![Vec::new(); hits.len()]; + let mut out = Vec::new(); + for h in 1..hits.len() { + let Some(p) = parent[h] else { + continue; + }; + let mut path = paths[p].clone(); + path.push(format!("{:?}{}", step(h, p), below[h])); + paths[h] = path.clone(); + out.push((hits[h].position, path)); + } + out +} + +/// Every pair of a projection with the depth of its group, `0` for the top. +fn pairs_by_depth(group: &ObservedGroup, depth: usize, out: &mut Vec<(usize, Tag16, Tag16)>) { + out.extend(group.pairs.iter().map(|&(s, vs)| (depth, s, vs))); + for element in &group.elements { + pairs_by_depth(element, depth + 1, out); + } +} + +/// Every needle and `alt` sent, by its query and role. +fn needles(world: &ObservedWorld) -> Vec> { + world + .queries + .iter() + .enumerate() + .flat_map(|(i, query)| match query { + ObservedQuery::Contains { needle, alt } => { + std::iter::once((format!("{i} needle"), needle)) + .chain(alt.iter().map(move |a| (format!("{i} alt"), a))) + .collect() + } + ObservedQuery::Equals { needle } => vec![(format!("{i} needle"), needle)], + _ => Vec::new(), + }) + .collect() +} + +/// Every value selector's colour: per row, the depths of the stored pairs +/// that carry it, and per needle (and `alt`), the depths of its pairs that +/// do. A value selector is a keyed hash of a path and a value, so its +/// selector is fixed and is not repeated here. +fn value_colours(world: &ObservedWorld) -> HashMap { + let mut stored: HashMap>> = HashMap::new(); + for (r, row) in world.rows.iter().enumerate() { + let mut pairs = Vec::new(); + pairs_by_depth(&row.q, 0, &mut pairs); + for (depth, _, vs) in pairs { + stored + .entry(vs) + .or_insert_with(|| vec![Vec::new(); world.rows.len()])[r] + .push(depth); + } + } + let mut sent: HashMap> = HashMap::new(); + for (what, needle) in needles(world) { + let mut pairs = Vec::new(); + pairs_by_depth(needle, 0, &mut pairs); + for (depth, _, vs) in pairs { + sent.entry(vs).or_default().push((what.clone(), depth)); + } + } + let mut out = HashMap::new(); + for vs in stored.keys().chain(sent.keys()) { + let mut rows = stored + .get(vs) + .cloned() + .unwrap_or_else(|| vec![Vec::new(); world.rows.len()]); + rows.iter_mut().for_each(|depths| depths.sort_unstable()); + let mut needles = sent.get(vs).cloned().unwrap_or_default(); + needles.sort_unstable(); + let _ = out.insert(*vs, format!("{rows:?}{needles:?}")); + } + out +} + +/// Ranks for `keys`, equal keys sharing one, in the keys' order. +fn dense(keys: &HashMap) -> HashMap { + let mut distinct: Vec<&K> = keys.values().collect(); + distinct.sort_unstable(); + distinct.dedup(); + keys.iter() + .map(|(s, k)| (*s, distinct.binary_search(&k).expect("listed"))) + .collect() +} + +fn classes(ranks: &HashMap) -> usize { + ranks.values().collect::>().len() +} + +/// A group's own pairs, by their selectors' and value selectors' ranks. +fn flat(group: &ObservedGroup, ranks: &HashMap) -> String { + let mut pairs: Vec<(usize, usize)> = group + .pairs + .iter() + .map(|(s, vs)| (ranks[s], ranks[vs])) + .collect(); + pairs.sort_unstable(); + format!("{pairs:?}") +} + +/// Where each pair of `group` and below it sits, recorded for its selector +/// and for its value selector: the projection, the depth, the other half of +/// the pair, and the group's own pairs, its parent's and its element +/// groups'. +fn places( + group: &ObservedGroup, + what: &str, + depth: usize, + parent: Option<&str>, + ranks: &HashMap, + out: &mut HashMap>, +) { + let mine = flat(group, ranks); + let mut elements: Vec = group.elements.iter().map(|g| flat(g, ranks)).collect(); + elements.sort_unstable(); + for (s, vs) in &group.pairs { + let around = format!("{what} {depth} {mine} {parent:?} {elements:?}"); + out.entry(*s) + .or_default() + .push(format!("value {} {around}", ranks[vs])); + out.entry(*vs) + .or_default() + .push(format!("selector {} {around}", ranks[s])); + } + for element in &group.elements { + places(element, what, depth + 1, Some(&mine), ranks, out); + } +} + +/// `ranks` split by where each name's pairs sit in `projections`, +/// repeatedly, until nothing more splits. +fn refine( + mut ranks: HashMap, + projections: &[Projection<'_>], +) -> HashMap { + loop { + let mut at: HashMap> = HashMap::new(); + for (what, projection) in projections { + places(projection, what, 0, None, &ranks, &mut at); + } + let keys: HashMap)> = ranks + .iter() + .map(|(s, rank)| { + let mut mine = at.remove(s).unwrap_or_default(); + mine.sort_unstable(); + (*s, (*rank, mine)) + }) + .collect(); + let next = dense(&keys); + if classes(&next) == classes(&ranks) { + return next; + } + ranks = next; + } +} + +/// Every row's `q`, by its row. +fn stored(world: &ObservedWorld) -> Vec> { + world + .rows + .iter() + .enumerate() + .map(|(r, row)| (format!("row {r}"), &row.q)) + .collect() +} + +/// Every name in the world (a selector or a value selector, stored or +/// sent) by what it shows at rest: a selector by, per row, whether the root +/// carries it, its entries and its `q` pairs; a value selector by its +/// colour; then both refined by the groups of `q` their pairs sit in. Names +/// nothing at rest tells apart share a rank. +fn rest_ranks( + world: &ObservedWorld, + terms: &HashMap, usize>, + colours: &HashMap, +) -> HashMap { + type AtRest = (bool, Vec, Vec<(usize, String)>); + let blank = || -> Vec { vec![(false, Vec::new(), Vec::new()); world.rows.len()] }; + let mut at_rest: HashMap> = HashMap::new(); + for (r, row) in world.rows.iter().enumerate() { + for (j, entry) in row.entries.iter().enumerate() { + let mine = &mut at_rest.entry(entry.s).or_insert_with(blank)[r]; + mine.0 |= j == 0; + mine.1.push(entry_key(entry, terms)); + } + let mut pairs = Vec::new(); + pairs_by_depth(&row.q, 0, &mut pairs); + for (depth, s, vs) in pairs { + at_rest.entry(s).or_insert_with(blank)[r] + .2 + .push((depth, colours[&vs].clone())); + } + } + let (selectors, values) = first_seen(world); + let mut rest: HashMap = HashMap::new(); + for s in selectors { + let mut mine = at_rest.get(&s).cloned().unwrap_or_else(blank); + for (_, entries, pairs) in &mut mine { + entries.sort_unstable(); + pairs.sort_unstable(); + } + let _ = rest.insert(s, (0, format!("{mine:?}"))); + } + for vs in values { + let _ = rest.insert(vs, (1, colours[&vs].clone())); + } + refine(dense(&rest), &stored(world)) +} + +/// Every name in the world by its rank (see the module docs for the +/// tiers), every one its own: `rest` first, then the queries that name it, +/// then where the walks find a selector's entries, then the needles' +/// groups as `q`'s were. +fn ranks( + world: &ObservedWorld, + terms: &HashMap, usize>, + walks: &[Walk], + rest: &HashMap, +) -> HashMap { + let walked = walk_tier(world, &addresses(world, walks, terms, rest)); + let mut sent: HashMap> = HashMap::new(); + for (i, query) in world.queries.iter().enumerate() { + match query { + ObservedQuery::Extract { s } => { + sent.entry(*s).or_default().push(format!("{i} extract")); + } + ObservedQuery::Order { s, op } => { + sent.entry(*s) + .or_default() + .push(format!("{i} order {}", terms[op])); + } + _ => {} + } + } + // A needle is sent in the walk's order, so a pair's place in it is + // part of what the database sees (`{"list": …, "name": …}` sends the + // two in their keys' byte order, which may be all that tells two + // selectors apart). + for (what, needle) in needles(world) { + let mut pairs = Vec::new(); + pairs_by_depth(needle, 0, &mut pairs); + for (place, (depth, s, vs)) in pairs.into_iter().enumerate() { + for name in [s, vs] { + sent.entry(name) + .or_default() + .push(format!("{what} {place} {depth}")); + } + } + } + let (selectors, values) = first_seen(world); + let seen: Vec = selectors.into_iter().chain(values).collect(); + // Its rank at rest, the queries that name it, where the walks find it. + type Full = (usize, Vec, Vec<(usize, Found)>); + let full: HashMap = seen + .iter() + .map(|s| { + let mut queries = sent.get(s).cloned().unwrap_or_default(); + queries.sort_unstable(); + let walks = walked.get(s).cloned().unwrap_or_default(); + (*s, (rest[s], queries, walks)) + }) + .collect(); + let mut projections = stored(world); + projections.extend(needles(world)); + // Refine by the projections' groups and by the walks, each walk's nodes + // described with the ranks so far, until neither splits anything: a + // query that tells two selectors apart tells apart the nodes the walks + // find beneath theirs. + let settle = |mut ranks: HashMap| loop { + let refined = refine(ranks, &projections); + let walked = walk_tier(world, &addresses(world, walks, terms, &refined)); + let keys: HashMap)> = refined + .iter() + .map(|(s, rank)| (*s, (*rank, walked.get(s).cloned().unwrap_or_default()))) + .collect(); + let next = dense(&keys); + if classes(&next) == classes(&refined) { + return next; + } + ranks = next; + }; + let mut ranks = settle(dense(&full)); + // Names still tied are ones nothing tells apart (`$.a.b` and `$.a.b.c` + // in a row `{"a": {"b": {"c": {}}}}`, at rest, or two values of one + // array that every row holds together). Each tie is broken once, for + // the entries and `q` alike, by setting the first of them apart and + // refining again, so that the choice carries to every name it bears + // on. + loop { + let mut by_rank: BTreeMap> = BTreeMap::new(); + for s in &seen { + by_rank.entry(ranks[s]).or_default().push(*s); + } + let Some(chosen) = by_rank + .values() + .find(|tied| tied.len() > 1) + .map(|tied| tied[0]) + else { + return ranks; + }; + let keys: HashMap = ranks + .iter() + .map(|(s, rank)| (*s, (*rank, *s != chosen))) + .collect(); + ranks = settle(dense(&keys)); + } +} + +/// Every selector and every value selector in the world, each in the order +/// it first appears: in the stored rows' entries, then their `q`, then the +/// queries. +fn first_seen(world: &ObservedWorld) -> (Vec, Vec) { + let mut selectors: Vec = Vec::new(); + let mut values: Vec = Vec::new(); + let note = |list: &mut Vec, name: Tag16| { + if !list.contains(&name) { + list.push(name); + } + }; + for row in &world.rows { + row.entries.iter().for_each(|e| note(&mut selectors, e.s)); + } + let mut pairs = Vec::new(); + for row in &world.rows { + pairs_by_depth(&row.q, 0, &mut pairs); + } + for (_, needle) in needles(world) { + pairs_by_depth(needle, 0, &mut pairs); + } + for (_, s, vs) in pairs { + note(&mut selectors, s); + note(&mut values, vs); + } + for query in &world.queries { + if let ObservedQuery::Extract { s } | ObservedQuery::Order { s, .. } = query { + note(&mut selectors, *s); + } + } + (selectors, values) +} + +/// A projection in the canonical order: its pairs by their selectors' and +/// value selectors' ranks, its element groups (each in the canonical order +/// itself) by their contents. +fn group(stored: &ObservedGroup, ranks: &HashMap) -> ObservedGroup { + let mut pairs = stored.pairs.clone(); + pairs.sort_by_key(|(s, vs)| (ranks[s], ranks[vs])); + let mut elements: Vec = + stored.elements.iter().map(|g| group(g, ranks)).collect(); + elements.sort_by_cached_key(|g| contents(g, ranks)); + ObservedGroup { pairs, elements } +} + +/// A canonical group's contents, as a string that sorts it among its +/// siblings. +fn contents(group: &ObservedGroup, ranks: &HashMap) -> String { + let elements: Vec = group.elements.iter().map(|g| contents(g, ranks)).collect(); + format!("{}{elements:?}", flat(group, ranks)) +} diff --git a/packages/stack-encrypt/tests/json_leakage/corpus.rs b/packages/stack-encrypt/tests/json_leakage/corpus.rs index 143148ba8..274484cfe 100644 --- a/packages/stack-encrypt/tests/json_leakage/corpus.rs +++ b/packages/stack-encrypt/tests/json_leakage/corpus.rs @@ -73,6 +73,9 @@ fn shape_queries(sealed_root: bool) -> Vec { Query::Contains(json!("a")), Query::Contains(json!({"hello": "world-1"})), Query::Contains(json!([{"x": 1}])), + // Sibling keys whose byte order is not their order as written, nor + // jsonb's: a needle is sent in the walk's order, so it shows which. + Query::Contains(json!({"a": "b", "a b": 1, "é": [], "0": {}})), Query::Equals("$.hello".into(), json!("world-1")), Query::Equals("$.nested.list[*].x".into(), json!(2)), Query::Order("$.number".into(), json!(0)), @@ -168,8 +171,11 @@ fn numbers() -> World { } /// A column with three wide top-level objects, each read whole, so that -/// whether members' ordinals follow their keys' order is decidable rather -/// than a coin toss (see `distinguishers::member_ordinals_follow_key_order`). +/// whether a read's subtree sits at consecutive stored slots is decidable +/// rather than a coin toss (`distinguishers::walked_subtrees_are_contiguous`), +/// and an array of twelve like objects, read whole, so that whether their +/// members' ordinals share one order is +/// (`distinguishers::member_order_shared_across_parents`). fn wide() -> World { let object = |prefix: &str, n: usize| { Value::Object( @@ -186,7 +192,8 @@ fn wide() -> World { }, docs: vec![ json!({"w1": object("a", 8), "w2": object("b", 8), "w3": object("c", 8), - "arr": [{"k": 1}, {"k": {"z": 2}, "m": [3]}]}), + "arr": [{"k": 1}, {"k": {"z": 2}, "m": [3]}], + "objs": (0..12).map(|i| json!({"p": i, "q": i, "r": i, "s": i})).collect::>()}), json!({"w1": object("a", 5), "w3": object("c", 8), "arr": [{"k": 4}]}), ], // `$.arr[*].k` at a bound of one: the probe `$.arr[1]` finds the @@ -197,6 +204,7 @@ fn wide() -> World { Query::subtree("$.w2"), Query::subtree("$.w3"), Query::subtree_with("$.arr[*].k", 1, 64), + Query::subtree("$.objs"), ], } } diff --git a/packages/stack-encrypt/tests/json_leakage/distinguishers.rs b/packages/stack-encrypt/tests/json_leakage/distinguishers.rs index 3f3abd8d2..75f2255ab 100644 --- a/packages/stack-encrypt/tests/json_leakage/distinguishers.rs +++ b/packages/stack-encrypt/tests/json_leakage/distinguishers.rs @@ -11,6 +11,16 @@ //! say), which changes a feature with probability below `n² / 2^64` for `n` //! tags in a world, about `10^-13` at the harness's sizes. //! +//! The entries are stored in a random order, so each distinguisher reads a +//! world in its canonical order ([`super::canonical`]): a row's root at +//! position `0` and its other entries, and every list of `q`, sorted by what +//! an observer sees of them without the stored order. A *position* below is +//! a place in that order, and a selector's name is its first appearance in +//! it. One feature reads the stored order itself, through the slot each +//! entry came from ([`walked_subtrees_are_contiguous`]), and like +//! [`member_order_shared_across_parents`] it carries its probability bound +//! in its doc comment. +//! //! What is deliberately not here, and why, is in the module docs //! (`mod.rs`, "Out of reach"). @@ -59,9 +69,7 @@ registry!( headers, projections, value_selector_partitions, - chain_head_runs, - selector_precedence, - depth_recovery, + walked_subtrees_are_contiguous, token_counts, bound_recovery, token_hits, @@ -70,7 +78,7 @@ registry!( hit_rules, ancestor_links, ordinal_patterns, - member_ordinals_follow_key_order, + member_order_shared_across_parents, token_derivability, subtree_results, transcripts, @@ -147,8 +155,10 @@ pub fn entry_counts(world: &ObservedWorld) -> Vec { world.rows.iter().map(|r| r.entries.len()).collect() } -/// Which entries carry a parent tag, by position. Catches a leak in which -/// entries are chain heads. +/// Which entries carry a parent tag, by position: with the selectors +/// ([`selector_pattern_across_rows`]), which paths are chain heads and how +/// many heads each row has. Catches a leak in which entries are chain +/// heads. pub fn parent_tag_presence(world: &ObservedWorld) -> Vec> { world .rows @@ -429,10 +439,10 @@ fn name_group( } } -/// Every row's projection `q`: its grouping by array element, the order of -/// its pairs, and which pairs share a selector or a value selector across -/// the column. Catches a value equality, a grouping or an order the profile -/// does not state. +/// Every row's projection `q`: its grouping by array element and which +/// pairs share a selector or a value selector across the column. Catches a +/// value equality or a grouping the profile does not state. (The stored +/// order of `q`'s lists is random; this reads them in the canonical order.) pub fn projections(world: &ObservedWorld) -> Vec { let mut selectors = selector_names(world); let mut values = Canon::new(); @@ -473,138 +483,6 @@ pub fn value_selector_partitions( out } -// ============================================================================= -// At rest: what the order of the entries says -// ============================================================================= - -/// The runs that start at each entry without a parent tag (other than the -/// first): under the sealed root these are the top-level fields, and since -/// everything beneath a field follows it, each run's length is that field's -/// subtree size. "The node count of each top-level field's subtree can be -/// read off a single row exactly." Parent tags are keyed by ordinal, so no -/// two entries share one and there are no equal-tag runs to read; this is -/// the contiguity structure there is. -pub fn chain_head_runs(world: &ObservedWorld) -> Vec> { - let mut canon = selector_names(world); - world - .rows - .iter() - .map(|row| { - let starts: Vec = (1..row.entries.len()) - .filter(|&j| row.entries[j].p.is_none()) - .collect(); - starts - .iter() - .enumerate() - .map(|(k, &start)| { - let end = starts.get(k + 1).copied().unwrap_or(row.entries.len()); - (canon.id(row.entries[start].s), end - start) - }) - .collect() - }) - .collect() -} - -/// For every two selectors that occur in one row, which of them came first -/// in the rows that hold both: `(a first, b first)`. Under a depth-first -/// order with members sorted by key, every two sibling members appear in one -/// order in every row, which is "the alphabetical order of sibling key -/// names"; a stored order independent of the keys would show both. -pub fn selector_precedence(world: &ObservedWorld) -> BTreeMap<(usize, usize), (bool, bool)> { - let mut canon = selector_names(world); - let mut out: BTreeMap<(usize, usize), (bool, bool)> = BTreeMap::new(); - for row in &world.rows { - let mut first: Vec = Vec::new(); - for entry in &row.entries { - let id = canon.id(entry.s); - if !first.contains(&id) { - first.push(id); - } - } - for (i, &a) in first.iter().enumerate() { - for &b in &first[i + 1..] { - let (key, a_first) = if a < b { - ((a, b), true) - } else { - ((b, a), false) - }; - let seen = out.entry(key).or_default(); - if a_first { - seen.0 = true; - } else { - seen.1 = true; - } - } - } - } - out -} - -/// An observer's reconstruction of every row's depth sequence from the -/// stored order alone, as the page says one can: "an observer comparing rows -/// learns" which selector's path sits under which. -/// -/// The attack: a selector's parent is among the selectors that precede it -/// in every row that holds it; of those it takes the one that itself has the -/// most such predecessors (the deepest candidate), and gives up where two -/// tie. An entry with no parent tag after the first is a top-level field -/// (depth 1). Each entry's depth is then its selector's chain of parents. -/// The attack is not always right (from order alone, a field that always -/// follows a sibling in every row reads as its child), and it does not need -/// to be: the feature is compared between worlds, and -/// `the_order_reveals_the_shape` checks that it is right on a corpus whose -/// rows vary. -pub fn depth_recovery(world: &ObservedWorld) -> Vec>> { - let mut canon = selector_names(world); - let rows: Vec> = world - .rows - .iter() - .map(|row| row.entries.iter().map(|e| canon.id(e.s)).collect()) - .collect(); - let mut candidates: HashMap> = HashMap::new(); - let mut heads: BTreeSet = BTreeSet::new(); - let mut roots: BTreeSet = BTreeSet::new(); - for (row, ids) in world.rows.iter().zip(&rows) { - for (k, &b) in ids.iter().enumerate() { - if k == 0 { - let _ = roots.insert(b); - } else if row.entries[k].p.is_none() { - let _ = heads.insert(b); - } - let before: BTreeSet = ids[..k].iter().copied().filter(|&a| a != b).collect(); - let _ = candidates - .entry(b) - .and_modify(|c| *c = c.intersection(&before).copied().collect()) - .or_insert(before); - } - } - let parent = |b: usize| -> Option { - let cands = candidates.get(&b)?; - let size = |a: &usize| candidates.get(a).map_or(0, BTreeSet::len); - let best = cands.iter().map(size).max()?; - let mut top = cands.iter().filter(|a| size(a) == best); - let first = *top.next()?; - top.next().is_none().then_some(first) - }; - let depth = |b: usize| -> Option { - let mut at = b; - // A chain of parents longer than there are selectors is a cycle. - for d in 0..=candidates.len() { - if roots.contains(&at) { - return Some(d); - } - if heads.contains(&at) { - return Some(d + 1); - } - at = parent(at)?; - } - None - }; - rows.iter() - .map(|ids| ids.iter().map(|&b| depth(b)).collect()) - .collect() -} - // ============================================================================= // Under a token: the database's walk, with the real hash // ============================================================================= @@ -613,12 +491,12 @@ pub fn depth_recovery(world: &ObservedWorld) -> Vec>> { /// self tag to position and parent tag to position, the last entry winning /// a repeated tag (`jsonb_object_agg` keeps the last value of a repeated /// key). -struct RowIndex { +pub(super) struct RowIndex { x: HashMap, p: HashMap, } -fn row_index(row: &ObservedRow) -> RowIndex { +pub(super) fn row_index(row: &ObservedRow) -> RowIndex { let mut index = RowIndex { x: HashMap::new(), p: HashMap::new(), @@ -636,13 +514,13 @@ fn row_index(row: &ObservedRow) -> RowIndex { /// One entry a walk from a token reaches. #[derive(Clone, Copy, Debug)] -struct Hit { - position: usize, +pub(super) struct Hit { + pub(super) position: usize, /// The position of the hit's parent, `None` for the node the token /// names. - parent: Option, + pub(super) parent: Option, /// The ordinal the hit was probed at under its parent. - ordinal: Option, + pub(super) ordinal: Option, depth: usize, /// The hit's own token, when the database has one: the token sent, for /// the node it names, or the one the member rule or the element rule @@ -650,7 +528,7 @@ struct Hit { /// and not descended into. token: Option, /// The rule that gave a hit below the named node its token. - rule: Option, + pub(super) rule: Option, /// The token of the hit's parent (for the named node, its own). parent_token: Tag16, } @@ -679,7 +557,7 @@ fn children(row: &ObservedRow, index: &RowIndex, token: &Tag16) -> Vec { /// below it, depth first, bounded as the crate's walk (`JsonDocument::subtree`) /// and the SQL's are: no deeper than the index seals, and no more hits than /// entries. -fn walk_with(row: &ObservedRow, index: &RowIndex, token: &Tag16) -> Vec { +pub(super) fn walk_with(row: &ObservedRow, index: &RowIndex, token: &Tag16) -> Vec { fn go( row: &ObservedRow, index: &RowIndex, @@ -973,57 +851,38 @@ pub fn ancestor_links(world: &ObservedWorld) -> Vec>> { .collect() } -/// What the ordinals a walk probes say, in the two forms that do not depend -/// on the keyed rank's random order: how many reached nodes have children -/// that all share one selector (an array's elements, or an object's one -/// member) whose ordinals do not run in stored order, and every case where -/// one parent token ranks two child selectors one way in one row and the -/// other way in another. Both are zero for the real scheme: an element's -/// ordinal is its position, and a member's rank is a keyed function of its -/// parent's token and its key. -pub fn ordinal_patterns(world: &ObservedWorld) -> (usize, Vec<(usize, usize, usize)>) { +/// Every case where one parent token ranks two child selectors one way in +/// one row and the other way in another, as the ordinals a walk probes +/// show them. Empty for the real scheme: an element's ordinal is its +/// position, and a member's rank is a keyed function of its parent's token +/// and its key, so one token ranks its members one way in every row. +pub fn ordinal_patterns(world: &ObservedWorld) -> Vec<(usize, usize, usize)> { let mut selectors = selector_names(world); let mut tokens = Canon::new(); - let mut out_of_order = 0; let mut ranked: BTreeMap<(usize, usize, usize), BTreeSet> = BTreeMap::new(); for rows in walks(world) { for (r, found) in rows.into_iter().enumerate() { let row = &world.rows[r]; for (_, hits) in found { - let mut by_parent: BTreeMap<(usize, Tag16), Vec> = BTreeMap::new(); - for hit in hits.into_iter().filter(|h| h.depth > 0) { - by_parent - .entry((hit.parent.expect("below the node"), hit.parent_token)) - .or_default() - .push(hit); - } - for ((_, parent_token), mut kids) in by_parent { - kids.sort_by_key(|h| h.position); + for ((_, parent_token), kids) in children_by_parent(hits) { let same = kids .windows(2) .all(|w| row.entries[w[0].position].s == row.entries[w[1].position].s); - if same - && kids - .iter() - .enumerate() - .any(|(i, h)| h.ordinal != Some(i as u32)) - { - out_of_order += 1; + if same { + continue; } - if !same { - let parent = tokens.id(parent_token); - for a in &kids { - for b in &kids { - let (sa, sb) = ( - selectors.id(row.entries[a.position].s), - selectors.id(row.entries[b.position].s), - ); - if sa < sb { - let _ = ranked - .entry((parent, sa, sb)) - .or_default() - .insert(a.ordinal < b.ordinal); - } + let parent = tokens.id(parent_token); + for a in &kids { + for b in &kids { + let (sa, sb) = ( + selectors.id(row.entries[a.position].s), + selectors.id(row.entries[b.position].s), + ); + if sa < sb { + let _ = ranked + .entry((parent, sa, sb)) + .or_default() + .insert(a.ordinal < b.ordinal); } } } @@ -1031,63 +890,155 @@ pub fn ordinal_patterns(world: &ObservedWorld) -> (usize, Vec<(usize, usize, usi } } } - let inconsistent = ranked + ranked .into_iter() .filter(|(_, seen)| seen.len() > 1) .map(|(key, _)| key) - .collect(); - (out_of_order, inconsistent) + .collect() } -/// Whether the members a token's holder finds directly under a token it was -/// sent carry ordinals in their stored order, which is their keys' sorted -/// order: the question decision 11 (the keyed member rank) is there to make -/// the answer to "no". Reported only where chance cannot answer it. +/// The hits of one walk below the node it names, grouped by their parent's +/// position and token, each group in the order of its ordinals. +fn children_by_parent(hits: Vec) -> BTreeMap<(usize, Tag16), Vec> { + let mut by_parent: BTreeMap<(usize, Tag16), Vec> = BTreeMap::new(); + for hit in hits.into_iter().filter(|h| h.depth > 0) { + by_parent + .entry((hit.parent.expect("below the node"), hit.parent_token)) + .or_default() + .push(hit); + } + for kids in by_parent.values_mut() { + kids.sort_by_key(|h| h.ordinal); + } + by_parent +} + +/// Whether two objects a token's holder walks, under two tokens, whose +/// nodes share a selector and whose members carry the same selectors, give +/// those members one order of ordinals: what a member's ordinal would do if +/// it ranked the key names. Decision 11's keyed rank, a PRF of the parent's +/// token and the key, is there to make the answer "no". With the stored +/// order random, the holder has no key order to set the ordinals against +/// (as it had before, where the stored order put the members in key order); +/// what it can still set them against is another object's. Reported only +/// where chance cannot answer it. /// -/// The bound: a member's ordinal is its rank under a keyed function of its -/// parent's token and its key, so for one token the members of its largest -/// observed object fall in key order with probability `1 / k!` for `k` -/// members, and distinct tokens are independent keyed inputs. All of them -/// fall in key order by chance with probability at most the product of -/// `1 / k!` over the tokens sent. The feature is `None` unless that product -/// is below `2^-40`; above it, real and simulated could disagree by chance. -/// Only objects directly under a token sent are counted, since which deeper -/// nodes share a derived token depends on the ranks themselves. -pub fn member_ordinals_follow_key_order(world: &ObservedWorld) -> Option { - let indexes: Vec = world.rows.iter().map(row_index).collect(); - let mut chance = 1.0f64; - let mut all_in_order = true; - let mut sent = BTreeSet::new(); - for (_, tokens, _) in subtree_queries(world) { - for t in tokens { - if !sent.insert(*t) { - continue; - } - let mut widest = 0usize; - for (row, index) in world.rows.iter().zip(&indexes) { - let mut kids: Vec = walk_with(row, index, t) - .into_iter() - .filter(|h| h.depth == 1) - .collect(); - let distinct: BTreeSet = - kids.iter().map(|h| row.entries[h.position].s).collect(); - if kids.len() < 2 || distinct.len() != kids.len() { - continue; +/// The bound: under one parent token the order of `m` members is a keyed +/// permutation, and distinct tokens are independent keyed inputs, so `t` +/// tokens over the same `m` member selectors give one order with +/// probability `(1 / m!)^(t − 1)`. Each token is counted once (its first +/// object of two or more members with distinct selectors, in the order the +/// queries, rows and tokens come), so the groups are independent and the +/// chance that every group agrees is the product. The feature is `None` +/// unless that product is below `2^-40`. The same path in two rows is one +/// token (tokens are column-wide), so it is the elements of an array of +/// like objects that decide it. +pub fn member_order_shared_across_parents(world: &ObservedWorld) -> Option { + let mut counted: BTreeSet = BTreeSet::new(); + let mut groups: BTreeMap<(Tag16, Vec), Vec>> = BTreeMap::new(); + for rows in walks(world) { + for (r, found) in rows.into_iter().enumerate() { + let row = &world.rows[r]; + for (_, hits) in found { + for ((parent, parent_token), kids) in children_by_parent(hits) { + // Children with pairwise distinct selectors are an + // object's members: an array's elements share one. + let order: Vec = + kids.iter().map(|h| row.entries[h.position].s).collect(); + let mut members = order.clone(); + members.sort_unstable(); + members.dedup(); + if members.len() < 2 + || members.len() != kids.len() + || !counted.insert(parent_token) + { + continue; + } + groups + .entry((row.entries[parent].s, members)) + .or_default() + .push(order); } - widest = widest.max(kids.len()); - kids.sort_by_key(|h| h.position); - if kids - .iter() - .enumerate() - .any(|(i, h)| h.ordinal != Some(i as u32)) - { - all_in_order = false; + } + } + } + let mut chance = 0.0f64; + let mut shared = true; + for ((_, members), orders) in &groups { + let factorial: f64 = (1..=members.len()).map(|k| (k as f64).ln()).sum(); + chance -= factorial * (orders.len() - 1) as f64; + shared &= orders.windows(2).all(|w| w[0] == w[1]); + } + (chance < -40.0 * 2f64.ln()).then_some(shared) +} + +/// Whether every subtree a walk links sits at consecutive stored slots, the +/// node its token names first: what a depth-first stored order gives every +/// subtree, and what the read check before the cascade required. This is +/// the one feature that reads the stored order, through the slot each +/// entry came from ([`ObservedRow::slots`]); every other reads the +/// canonical order. Reported only where chance cannot answer it. +/// +/// The bound: a row of `n` entries stores its root at slot `0` and the +/// others at a uniformly random permutation of the `N = n − 1` slots after +/// it. Take `k` pairwise disjoint walked subtrees of sizes `m_1 … m_k`, none +/// holding the root: they all land as runs that start at their named nodes +/// with probability `(N − Σm + k)! · Π (m_i − 1)! / N!` (each run is a +/// block whose first entry is fixed). A walked subtree that holds the root +/// (its named node is `$` with the root open) must fill the `m − 1` slots +/// after it, which multiplies in `(m − 1)!` and takes those slots out of +/// `N` first. That every walked subtree is a run implies that the chosen +/// disjoint ones are, so the chosen family's probability bounds it; each +/// row takes its largest walked subtrees greedily, and rows are drawn +/// independently, so the chance for the whole world is at most the product +/// over the rows. The feature is `None` unless that product is below +/// `2^-40`. +pub fn walked_subtrees_are_contiguous(world: &ObservedWorld) -> Option { + let ln_factorial = |n: usize| -> f64 { (2..=n).map(|k| (k as f64).ln()).sum() }; + let mut per_row: Vec)>> = vec![Vec::new(); world.rows.len()]; + for rows in walks(world) { + for (r, found) in rows.into_iter().enumerate() { + for (_, hits) in found { + let first = hits[0].position; + let set: BTreeSet = hits.iter().map(|h| h.position).collect(); + if set.len() > 1 && !per_row[r].iter().any(|(_, s)| *s == set) { + per_row[r].push((first, set)); } } - chance /= (1..=widest).map(|k| k as f64).product::(); } } - (chance < 2f64.powi(-40)).then_some(all_in_order) + let mut runs = true; + let mut chance = 0.0f64; + for (row, mut subtrees) in world.rows.iter().zip(per_row) { + for (first, set) in &subtrees { + let slots: BTreeSet = set.iter().map(|&j| row.slots[j]).collect(); + let start = row.slots[*first]; + runs &= slots.len() == set.len() + && slots.first() == Some(&start) + && slots.last() == Some(&(start + set.len() - 1)); + } + subtrees.sort_by_key(|(_, set)| std::cmp::Reverse(set.len())); + let mut taken: BTreeSet = BTreeSet::new(); + let mut free = row.entries.len().saturating_sub(1); + let mut blocks = Vec::new(); + let mut favourable = 0.0; + for (_, set) in subtrees { + if !set.is_disjoint(&taken) { + continue; + } + taken.extend(set.iter().copied()); + favourable += ln_factorial(set.len() - 1); + if set.contains(&0) { + free = free.saturating_sub(set.len() - 1); + } else { + blocks.push(set.len()); + } + } + let singles = free.saturating_sub(blocks.iter().sum::()); + favourable += ln_factorial(singles + blocks.len()); + chance += favourable - ln_factorial(row.entries.len().saturating_sub(1)); + } + (chance < -40.0 * 2f64.ln()).then_some(runs) } /// Which subtree reads' tokens the database can derive from which others': diff --git a/packages/stack-encrypt/tests/json_leakage/mod.rs b/packages/stack-encrypt/tests/json_leakage/mod.rs index 2ac95900c..17513c5e3 100644 --- a/packages/stack-encrypt/tests/json_leakage/mod.rs +++ b/packages/stack-encrypt/tests/json_leakage/mod.rs @@ -17,12 +17,18 @@ //! ported; `docs/plans/2026-10-25-json-tree-leakage-before-and-after.md` //! compares the two profiles clause by clause. //! +//! Since then a document's entries are stored in a uniformly random order +//! drawn when it is sealed, the root at slot `0`, and `q`'s lists are +//! shuffled too ([`profile::StoredOrder`]); and each entry's AAD binds its +//! children's AEAD tags (the cascade), which leaves everything stored as it +//! was and so changes no clause. +//! //! # The pieces //! //! ```text //! documents, queries ──► profile::leak ──► Leakage ──► simulator::simulate ──► wire JSON ─┐ //! (plaintext) (L: no key) (+ seeded randomness) │ -//! ├─► observe ──► ObservedWorld ──► distinguishers +//! ├─► observe ──► canonical ──► distinguishers //! documents, queries ──► real::world ─────────────────────────────────────────► wire JSON ─┘ //! (the crate, under the fake key source) //! ``` @@ -38,6 +44,8 @@ //! - [`simulator`] builds a world in the wire form from a //! [`profile::Leakage`] and randomness, with no plaintext and no key: //! random bytes for every name, random ciphertexts of the stated length, +//! each row's entries at a random permutation of their slots with the root +//! first and `q`'s lists shuffled, as the crate stores them, //! ordering terms with CLLW's layout under a key of its own, over codes //! that have the stated order, lengths and shared prefixes and nothing //! else of the values, and the chain programmed @@ -48,12 +56,20 @@ //! through the crate's public API, and serializes them. //! - [`observe`] parses either world's wire JSON into one //! [`observe::ObservedWorld`]. -//! - [`distinguishers`] are pure functions of an observed world: what an -//! observer holding the column and the query terms could compute. Each -//! feature is exact (byte strings renamed to integers in order of first -//! appearance), so a real and a simulated world are compared for equality, -//! with no statistics. One feature needs a probability bound and carries it -//! in its doc comment ([`distinguishers::member_ordinals_follow_key_order`]). +//! - [`canonical`] puts a world's entries and `q`'s lists in an order an +//! observer computes without the stored order, keeping each entry's stored +//! slot beside it. The stored order is random on both sides, independently, +//! so a feature that named an entry by its slot would tell a faithful +//! simulation from the real scheme by chance alone. +//! - [`distinguishers`] are pure functions of an observed world, read in the +//! canonical order: what an observer holding the column and the query +//! terms could compute. Each feature is exact (byte strings renamed to +//! integers in order of first appearance), so a real and a simulated world +//! are compared for equality, with no statistics. Two features need a +//! probability bound and carry it in their doc comments +//! ([`distinguishers::walked_subtrees_are_contiguous`], the one that reads +//! the stored order, and +//! [`distinguishers::member_order_shared_across_parents`]). //! - [`mutants`] are profiles with one clause dropped or coarsened. //! - [`corpus`] holds the handcrafted columns and a proptest generator of //! random ones over a shared key vocabulary. @@ -76,10 +92,11 @@ //! [`profile::Clauses::as_published_in_1363`] turns it off, and a test //! holds that the result is then told from the real scheme, by the //! distinguisher that names it and the three its absence breaks. -//! - **Three readings.** The page states the key order (UTF-8 byte order), -//! that strings order folded, and that a number's equality under its term -//! is as a 64-bit float; a profile read the other way on any of them is -//! told from the real scheme, so each sentence is load-bearing. +//! - **Three readings.** The key order (UTF-8 byte order, which a needle's +//! pairs are sent in), that strings order folded, and that a number's +//! equality under its term is as a 64-bit float; a profile read the other +//! way on any of them is told from the real scheme, so each is +//! load-bearing. //! - **The calibration.** Each mutant profile drops one clause. The test //! expects some distinguisher to tell each mutant's world from the real //! one, and prints a table of which (run with `--no-capture`). A mutant @@ -88,15 +105,14 @@ //! clauses describe, so that the positive control passing means something. //! - **The harness itself.** The chain functions agree with the crate's own //! walk (`JsonDocument::subtree`) for reads through members, positions, -//! wildcards and `$`; the order-only attack in -//! [`distinguishers::depth_recovery`] recovers every row's depth sequence -//! on a column whose rows vary, which is the page's claim that the stored -//! order reveals each row's shape; and on the real scheme a wildcard -//! read's token list gives its bound and its probes away -//! (`a_read_gives_away_its_bound_and_its_probes`). Two features no mutant -//! can reach are shown to fire: `cross_kind_equalities` on a world whose -//! token is a stored selector, and `token_derivability` on a read of a -//! position past every row's array. +//! wildcards and `$`; one document sealed four times stores its root at +//! slot `0` each time and the rest of its entries, and `q`'s pairs, in a +//! fresh order each time (`the_order_is_drawn_afresh_with_the_root_first`); +//! and on the real scheme a wildcard read's token list gives its bound and +//! its probes away (`a_read_gives_away_its_bound_and_its_probes`). Two +//! features no mutant can reach are shown to fire: `cross_kind_equalities` +//! on a world whose token is a stored selector, and `token_derivability` +//! on a read of a position past every row's array. //! //! # What the profile found //! @@ -153,18 +169,39 @@ //! parts of a ciphertext's length ([`profile::ciphertext_len`]). //! //! The two issues the profile was first built to check: the stored order -//! still reveals each row's shape (#1357; the order clause is load-bearing -//! and the depth attack works), and a subtree read no longer hands over -//! its top-level field (#1360): the mutant that sends the head's token as -//! the read before #1363 did (`ancestor_token_sent`) is told apart by -//! `ancestor_links`, `token_reach` and the token counts. +//! revealed each row's shape (#1357; the order clause was load-bearing and +//! an order-only attack recovered every row's depth sequence), and a +//! subtree read no longer hands over its top-level field (#1360): the +//! mutant that sends the head's token as the read before #1363 did +//! (`ancestor_token_sent`) is told apart by `ancestor_links`, `token_reach` +//! and the token counts. #1357 is now closed by the random stored order: +//! the clause says the order shows nothing beyond the root's place, the +//! positive control holds it, and the mutant that states the depth-first +//! order before it (`stored_order_depth_first`) is told apart by +//! `walked_subtrees_are_contiguous`. +//! +//! Against the page as the random order leaves it: +//! +//! 3. **A needle still shows the key order.** A containment or equality +//! needle is sent in the walk's order, unshuffled, an object's members +//! in their keys' byte order, so a needle with two sibling keys tells the +//! database which sorts first, at paths whose selectors it also sees in +//! the stored rows. The stored `q` is shuffled; the needle is not. The +//! profile states it ([`profile::GroupLeakage::pairs`], the key-order +//! reading test, the mutant `needle_pairs_shuffled`, caught by +//! `transcripts`), and so does the page, under containment. Shuffling a +//! needle on the client, which the server reads as a set, would close +//! it; that is left to a decision of its own. //! //! # When the format changes //! //! `L` and the real-side adapter are expected to change with the format, //! and with the leakage page; the distinguishers mostly should not (the //! port to #1363 kept 26 of the 27 the profile had, renamed one and added -//! eight). To add or change a clause: +//! eight; the port to the random stored order read them all in the +//! canonical order, retired the three that read only the stored order, +//! replaced one that set member ordinals against it, and added the one that +//! now reads it). To add or change a clause: //! //! 1. Change the page first, or at least decide what it will say. //! 2. Change [`profile`]: a field on the leakage type, with the page's @@ -184,6 +221,12 @@ //! goes in marked as such in [`profile::Clauses`], with a test like the //! one for the hit rule, so the gap stays visible until the page closes //! it. +//! - **Positive control fails on a feature that names positions, and the +//! real and simulated features differ by a swap of two names.** Suspect +//! the canonical order before the profile: two entries (or selectors, or +//! values) it tied are told apart by something its descriptions leave +//! out. Add that to the description in [`canonical`]. The real side's +//! stored order is fresh on every run, so such a failure can come and go. //! - **Calibration fails for a mutant nothing noticed.** Either the clause //! is not load-bearing on the calibration columns (add a column that //! exercises it, as the wide-objects column does for member ranks and @@ -198,18 +241,16 @@ //! That is what happened when this module first ran against #1363's //! format. //! -//! # A tension this module states and does not resolve +//! # A tension this module stated, now resolved //! -//! PR #1358's read check (`JsonSubtreeKey::open`) refuses a walk's rows -//! unless each subtree's rows sit at consecutive positions `j` in `sv`, which -//! is what a subtree is in a depth-first stored order, and `j` is bound into -//! every ciphertext's AAD. #1363 keeps it (its decision 8: "each deeper row -//! must sit at the next consecutive `j` within its subtree"). That -//! consecutiveness is exactly the property #1357 wants gone: a stored order -//! independent of the shape (a keyed permutation of `sv`, say) would -//! scatter a subtree's rows. The check and "order independent of shape" -//! cannot both hold as written: one of them has to give, and which is the -//! owner's decision. +//! PR #1358's read check (`JsonSubtreeKey::open`) refused a walk's rows +//! unless each subtree's rows sat at consecutive positions `j` in `sv`, +//! which is what a subtree is in a depth-first stored order, and #1363 kept +//! it; that consecutiveness was exactly what #1357 wanted gone. The cascade +//! resolved it: each entry's AAD binds its children's AEAD tags, so the +//! client checks a read by the chain and the tags rather than by position, +//! the consecutive-position rule is gone, and the stored order is random. +//! The AAD still binds each entry's slot. //! //! # Out of reach //! @@ -237,9 +278,12 @@ //! jsonb's (`mixed_kinds_jsonb_order`) is caught because the test key put //! the strings below the numbers. //! - **The keyed member rank as a permutation.** Under a token, a member's -//! ordinal is a keyed rank. Its relation to the stored (key) order is a -//! random permutation, decidable exactly only where enough wide objects sit -//! under the tokens sent; elsewhere the feature reports nothing. +//! ordinal is a keyed rank, a random permutation of the members. Whether +//! two tokens' objects share one order is decidable exactly only where +//! enough like objects (an array's elements) sit under the tokens sent; +//! elsewhere the feature reports nothing. Likewise whether a walked +//! subtree sits at consecutive stored slots, where the subtrees are too +//! small for chance to rule it out. //! - **The primitives.** Ciphertexts beyond their length, selectors, value //! selectors, head tokens and unlinked self tags beyond their equality //! patterns: the profile assumes AES-GCM, the PRFs and the hash do their @@ -260,19 +304,24 @@ //! //! Every simulation and every mutant draws from one seeded generator, and //! the random columns come from a seeded proptest runner. The real side -//! draws data keys and nonces from the operating system; those reach a -//! feature only through equality patterns, which change only on a collision -//! of random bytes, below `n² / 2^64` for `n` eight-byte tags in a column -//! (about `10^-13` here). The fake key source's index key is fixed, so the -//! real member ranks are too. +//! draws data keys, nonces and every document's stored order from the +//! operating system. The keys and nonces reach a feature only through +//! equality patterns, which change only on a collision of random bytes, +//! below `n² / 2^64` for `n` eight-byte tags in a column (about `10^-13` +//! here). The stored order reaches none but through the canonical order, +//! which does not depend on it (ties it leaves are between entries no +//! feature tells apart), and through `walked_subtrees_are_contiguous`, +//! whose answer is fixed but with probability below `2^-40`. The fake key +//! source's index key is fixed, so the real member ranks are too. //! //! The profile sorts members itself rather than trusting `serde_json`'s //! iteration order, so a build that unified serde_json's `preserve_order` -//! feature on (which would store the application's field order, #1357) fails -//! the positive control on the handcrafted rows whose keys are written out -//! of order. +//! feature on (which would send a needle in the application's field order) +//! fails the positive control on the handcrafted needle whose keys are +//! written out of order. pub mod calibration; +pub mod canonical; pub mod chain; pub mod corpus; pub mod distinguishers; diff --git a/packages/stack-encrypt/tests/json_leakage/mutants.rs b/packages/stack-encrypt/tests/json_leakage/mutants.rs index 661c971c7..dfe01fc2a 100644 --- a/packages/stack-encrypt/tests/json_leakage/mutants.rs +++ b/packages/stack-encrypt/tests/json_leakage/mutants.rs @@ -10,7 +10,8 @@ //! mutant is "the profile without this clause". A few state a format the //! code does not have, to show the distinguishers would see it: the //! pre-#1363 read's top-level token (`ancestor_token_sent`), the pre-#1363 -//! member rule (`member_tokens_carry_ordinal`). +//! member rule (`member_tokens_carry_ordinal`), the depth-first stored +//! order before the random one (`stored_order_depth_first`). use std::collections::{BTreeMap, HashMap}; @@ -19,7 +20,7 @@ use proptest::test_runner::TestRng; use super::profile::{ EntryLeakage, GroupLeakage, KindOrder, Leakage, OrderKind, Placement, QueryLeakage, ReadLeak, - RowLeakage, Rule, Step, TokenName, TokenStep, AEAD_OVERHEAD, + RowLeakage, Rule, Step, StoredOrder, TokenName, TokenStep, AEAD_OVERHEAD, }; /// One mutant: its name, the clause it drops (a field or sentence of the @@ -86,34 +87,6 @@ fn every_read(leakage: &mut Leakage, mut f: impl FnMut(&mut Vec)) { } } -/// Reorder a row's entries: `order[k]` is the old position of the entry now -/// at `k`. Parents and every query's positions follow. -fn reorder(leakage: &mut Leakage, r: usize, order: &[usize]) { - let mut new_at = vec![0; order.len()]; - for (k, &old) in order.iter().enumerate() { - new_at[old] = k; - } - let row = &mut leakage.rows[r]; - let old = std::mem::take(&mut row.entries); - row.entries = order - .iter() - .map(|&o| EntryLeakage { - parent: old[o].parent.map(|p| new_at[p]), - ..old[o].clone() - }) - .collect(); - every_read(leakage, |reads| { - for read in reads { - if let Some(at) = &mut read.hits[r] { - *at = new_at[*at]; - } - for at in &mut read.reach[r] { - *at = new_at[*at]; - } - } - }); -} - fn children(row: &RowLeakage) -> Vec> { let mut kids = vec![Vec::new(); row.entries.len()]; for (i, entry) in row.entries.iter().enumerate() { @@ -240,35 +213,11 @@ fn key_tag_shortened(l: &mut Leakage, _: &mut TestRng) { l.key_tag_len = 16; } -fn order_random_permutation(l: &mut Leakage, rng: &mut TestRng) { - for r in 0..l.rows.len() { - let mut order: Vec = (1..l.rows[r].entries.len()).collect(); - shuffle(&mut order, rng); - order.insert(0, 0); - reorder(l, r, &order); - } -} - -/// Still depth-first, but an object's members in random order rather than -/// sorted by key. -fn members_unsorted(l: &mut Leakage, rng: &mut TestRng) { - for r in 0..l.rows.len() { - let kids = children(&l.rows[r]); - let mut order = Vec::new(); - let mut stack = vec![0]; - while let Some(i) = stack.pop() { - order.push(i); - let mut mine = kids[i].clone(); - if mine - .iter() - .all(|&k| l.rows[r].entries[k].step == Step::Member) - { - shuffle(&mut mine, rng); - } - stack.extend(mine.into_iter().rev()); - } - reorder(l, r, &order); - } +/// The stored order before the random one: depth-first, the root first, +/// an object's members in key order, and `q` in the same walk's order, which +/// gave every row's tree away (#1357). +fn stored_order_depth_first(l: &mut Leakage, _: &mut TestRng) { + l.stored_order = StoredOrder::DepthFirst; } fn parent_tag_on_every_entry(l: &mut Leakage, _: &mut TestRng) { @@ -335,12 +284,6 @@ fn projection_flat(l: &mut Leakage, _: &mut TestRng) { } } -fn projection_shuffled(l: &mut Leakage, rng: &mut TestRng) { - for row in &mut l.rows { - groups_mut(&mut row.projection, &mut |g| shuffle(&mut g.pairs, rng)); - } -} - fn ordering_terms_absent(l: &mut Leakage, _: &mut TestRng) { for row in &mut l.rows { for entry in &mut row.entries { @@ -832,6 +775,18 @@ fn needle_flat(l: &mut Leakage, _: &mut TestRng) { } } +/// A needle's pairs sent in a random order, as a stored `q`'s are stored: +/// what the profile would say if a needle's order revealed nothing. The +/// client sends a needle in the walk's order, an object's members in key +/// byte order, so a needle with two sibling keys tells their byte order. +fn needle_pairs_shuffled(l: &mut Leakage, rng: &mut TestRng) { + for query in &mut l.queries { + if let QueryLeakage::Contains { needle, .. } | QueryLeakage::Equals { needle } = query { + groups_mut(needle, &mut |g| shuffle(&mut g.pairs, rng)); + } + } +} + fn needle_alt_dropped(l: &mut Leakage, _: &mut TestRng) { for query in &mut l.queries { if let QueryLeakage::Contains { alt, .. } = query { @@ -881,8 +836,7 @@ mutants!( entry_count_padded => "RowLeakage::entries (\"the number of entries\")", parents_lifted => "EntryLeakage::parent (each node's parent, which a token's walk links)", element_ordinals_reversed => "EntryLeakage::step (\"an element's index is the ordinal it is found at\")", - order_random_permutation => "RowLeakage::entries (the stored order)", - members_unsorted => "RowLeakage::entries (\"an object's members in the byte order of their key names\")", + stored_order_depth_first => "Leakage::stored_order (\"stored in a random order […] with the root first\"): the depth-first order before", parent_tag_on_every_entry => "EntryLeakage::has_parent_tag", parent_tag_on_no_entry => "EntryLeakage::has_parent_tag", self_tags_absent => "EntryLeakage::self_tag (\"the self tag on every entry\")", @@ -893,7 +847,6 @@ mutants!( value_selectors_all_distinct => "GroupLeakage::pairs (value selector equality)", value_selectors_path_only => "GroupLeakage::pairs (\"a keyed hash of the path and the value\")", projection_flat => "GroupLeakage::elements (\"nested by array element\")", - projection_shuffled => "GroupLeakage::pairs (\"in the same order\")", ordering_terms_absent => "EntryLeakage::ordering (presence)", ordering_terms_unordered => "Leakage::ordering_ranks (\"the order of that leaf's value among every row\")", ordering_terms_all_distinct => "OrderingLeak::class (\"which rows hold equal values\")", @@ -923,6 +876,7 @@ mutants!( token_list_padded => "QueryLeakage::Subtree::reads (the tokens' number, which gives N away)", budget_not_sent => "QueryLeakage::Subtree::budget (the budget, in the clear)", needle_flat => "QueryLeakage::Contains::needle (\"its element grouping\")", + needle_pairs_shuffled => "GroupLeakage::pairs of a needle (sent in the walk's order, members in key byte order)", needle_alt_dropped => "QueryLeakage::Contains::alt", bound_rank_random => "QueryLeakage::Order::bound (\"the bound's place in the order of the stored values\")", ); diff --git a/packages/stack-encrypt/tests/json_leakage/observe.rs b/packages/stack-encrypt/tests/json_leakage/observe.rs index a5998d82a..46091d4db 100644 --- a/packages/stack-encrypt/tests/json_leakage/observe.rs +++ b/packages/stack-encrypt/tests/json_leakage/observe.rs @@ -9,6 +9,10 @@ //! [`ObservedWorld::from_wire`]. A simulator that wrote a field the real form //! lacks, or the other way round, shows in [`ObservedRow::fields`] and //! [`ObservedEntry::fields`]. +//! +//! A world is parsed in its stored order. The distinguishers read it in the +//! canonical order ([`super::canonical`]), which keeps each entry's stored +//! slot beside it ([`ObservedRow::slots`]). use base64ct::{Base64, Encoding}; use serde_json::Value; @@ -61,6 +65,10 @@ pub struct ObservedRow { pub chars: usize, pub header: ObservedHeader, pub entries: Vec, + /// The slot in `sv` each entry is stored at: `0, 1, 2, …` as parsed, + /// and wherever the canonical order took each entry from after + /// [`super::canonical::canonical`]. + pub slots: Vec, pub q: ObservedGroup, /// Every key path inside `q` ([`key_paths`]). pub q_fields: Vec, @@ -237,16 +245,18 @@ fn chars(value: &Value) -> usize { } fn row(value: &Value) -> ObservedRow { + let entries: Vec = value["sv"] + .as_array() + .expect("`sv` is a list") + .iter() + .map(entry) + .collect(); ObservedRow { fields: keys(value), chars: chars(value), header: header(&b64(&value["h"])), - entries: value["sv"] - .as_array() - .expect("`sv` is a list") - .iter() - .map(entry) - .collect(), + slots: (0..entries.len()).collect(), + entries, q: group(&value["q"]), q_fields: key_paths(&value["q"]), } diff --git a/packages/stack-encrypt/tests/json_leakage/profile.rs b/packages/stack-encrypt/tests/json_leakage/profile.rs index bffc691b1..aa849acc3 100644 --- a/packages/stack-encrypt/tests/json_leakage/profile.rs +++ b/packages/stack-encrypt/tests/json_leakage/profile.rs @@ -28,6 +28,13 @@ //! (`{s, c, p?, op?}` entries, one head token and plaintext gates per read); //! `docs/plans/2026-10-25-json-tree-leakage-before-and-after.md` records //! what changed between the two. +//! +//! Since then a document's entries are stored in a uniformly random order +//! drawn when it is sealed, the root first, and `q`'s lists are shuffled +//! the same way ([`Leakage::stored_order`]); each entry's AAD binds its +//! children's AEAD tags (the cascade), which changes nothing stored. The +//! profile numbers a row's nodes depth-first ([`flatten`]) for its own +//! bookkeeping: that numbering is not the stored order. use std::collections::{BTreeMap, HashMap}; use std::hash::Hash; @@ -105,8 +112,11 @@ impl Query { } /// Which sort "the byte order of their key names' UTF-8 encodings" means. -/// The page now says it; the other reading is kept to show the sentence is -/// load-bearing. +/// The stored order no longer shows it ([`StoredOrder::Random`]), but a +/// containment or equality needle is still sent in the walk's order, an +/// object's members in this order ([`GroupLeakage::pairs`]), so a needle +/// with two sibling keys tells the database their byte order. The other +/// reading is kept to show that is load-bearing. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum KeyOrder { /// Byte order of the UTF-8 key: serde_json's `Map` without @@ -146,8 +156,8 @@ pub enum NumberOrder { /// readings the page could leave open. The default is what the code needs. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct Clauses { - /// How "an object's members in the byte order of their key names" - /// sorts. + /// How an object's members sort in the walk's order, which is the order + /// a needle's pairs are sent in. pub key_order: KeyOrder, /// How a string's ordering term orders. pub string_order: StringOrder, @@ -206,7 +216,8 @@ pub type SelfTagId = usize; #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum Step { Root, - /// An object member: its ordinal is its keyed rank among its siblings. + /// An object member: its ordinal is its keyed rank among its siblings, + /// under its parent's token. Member, /// An array element at this position, which is its ordinal. Element(u64), @@ -280,20 +291,20 @@ pub struct EntryLeakage { /// array's selector, so `$.tags[0]` and `$.tags[1]` look alike, but each /// element is its own entry." pub selector: SelId, - /// The shape: the position in [`RowLeakage::entries`] of this node's - /// parent. "Which selector's path sits under which is not written down, - /// but it is the same in every row, so an observer comparing rows learns - /// it, and with it each row's parent and child relations and the node - /// count of every subtree." The simulator uses it to place the chain - /// below a token a query sends; at rest the order clause - /// ([`RowLeakage::entries`]) is what shows it. + /// The shape: the index in [`RowLeakage::entries`] of this node's + /// parent. Not revealed at rest: the stored order is random + /// ([`Leakage::stored_order`]) and no stored field links an entry to + /// another, so the simulator reads it only below a token a query sends, + /// to place the chain the token links ([`ReadLeak::reach`]): "see how + /// the nodes beneath it nest, link each one to its parent". pub parent: Option, /// The node's ordinal under its parent, which a token's holder sees in /// the ordinals it probes: "each element's index: […] an element's index /// is the ordinal it is found at". A member's ordinal is its keyed rank, /// which the page does not mention; under one token it is a fixed - /// permutation of the members' key order that the holder sees and - /// cannot read (`member_ordinals_follow_key_order`). + /// permutation of the members that the holder sees and cannot read, and + /// under two tokens two unrelated ones + /// (`member_order_shared_across_parents`). pub step: Step, /// The rule this node's token takes under its parent's: "whether each /// node beneath is an object member or an array element […] the @@ -332,11 +343,13 @@ pub struct EntryLeakage { /// One group of a projection, stored (`q`) or sent (a needle). #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct GroupLeakage { - /// "One `{s, vs}` pair per node, […] in the same order": the pairs of the - /// nodes that sit directly in this group, in the walk's order. - /// "Whether two rows hold the same value at the same path, through the - /// value selector `vs`, a keyed hash of the path and the value (for an - /// object or an array, of its kind alone)." + /// "One `{s, vs}` pair per node": the pairs of the nodes that sit + /// directly in this group, in the walk's order, an object's members in + /// [`Clauses::key_order`]. A needle is sent in this order; a stored + /// `q`'s lists are stored shuffled ([`Leakage::stored_order`]). "Whether + /// two rows hold the same value at the same path, through the value + /// selector `vs`, a keyed hash of the path and the value (for an object + /// or an array, of its kind alone)." pub pairs: Vec<(SelId, ValueId)>, /// "Nested by array element […] Which nodes sit inside the same array /// element, through the nesting, for every element that is an object or @@ -356,10 +369,12 @@ pub struct RowLeakage { /// data key's identifier (unique to the row)". A different name in every /// row. The `iv` also salts every parent and self tag in the row. pub data_key: usize, - /// "The number of entries", and the entries in their stored order: - /// "Entries are stored in depth-first order, the root first, each node - /// followed by everything beneath it, and an object's members in the - /// byte order of their key names' UTF-8 encodings." + /// "The number of entries", one per node, numbered as [`flatten`] walks + /// the document (depth-first, the root first). The numbering is the + /// profile's own: the entries are stored in the order + /// [`Leakage::stored_order`] says, and every index into this list + /// ([`EntryLeakage::parent`], [`ReadLeak::hits`], [`ReadLeak::reach`]) + /// names a node, not a slot. pub entries: Vec, /// The containment projection `q`. pub projection: GroupLeakage, @@ -450,6 +465,30 @@ pub enum QueryLeakage { }, } +/// The order a row's entries and its projection's lists are stored in. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum StoredOrder { + /// The page: "The entries of `sv` are stored with the root first, where + /// the database finds the sealed root, and every other entry in a + /// uniformly random order drawn when the row is sealed and then + /// discarded. [...] it shows nothing beyond the root's place", and "The + /// pairs of each group and the groups of each list are stored in a + /// random order, so the projection does not restate the document's + /// order". Slot `0` holds the + /// root; slots `1..n` hold the other nodes in a uniformly random + /// permutation; every list of `q` is a uniformly random permutation of + /// its pairs or groups. What stays visible is the rest of this profile: + /// the selectors a row holds and how many of each, `q`'s grouping by + /// array element, the entries without a parent tag (the chain heads), + /// each ciphertext's length and the ordering terms. + Random, + /// The order before: depth-first, the root first, each node followed by + /// everything beneath it, an object's members in key order, and `q` in + /// the same walk's order. It gave every row's tree away (#1357). Only a + /// mutant states it now. + DepthFirst, +} + /// The leakage of a whole column: its rows and its queries, over one shared /// set of opaque names. #[derive(Clone, Debug, PartialEq, Eq)] @@ -457,6 +496,8 @@ pub struct Leakage { /// "The keyset the data key was minted under […] the same for every /// column it seals": one keyset per column, and the ZeroKMS tag's length. pub key_tag_len: usize, + /// The order every row's entries and `q` are stored in. + pub stored_order: StoredOrder, pub rows: Vec, pub queries: Vec, /// The rank of each [`OrderClass`] among the classes of its kind: "the @@ -743,7 +784,7 @@ fn canonical(value: &Value) -> Vec { } } -/// One node of a document in its stored order. +/// One node of a document, as [`flatten`] numbers it. pub struct Node<'a> { pub value: &'a Value, pub positional: Vec, @@ -755,9 +796,10 @@ pub struct Node<'a> { pub container_element: bool, } -/// A document's nodes in the order the page says they are stored: -/// depth-first, the root first, an object's members in `order`, an array's -/// elements in position order. +/// A document's nodes in the walk's order: depth-first, the root first, an +/// object's members in `order`, an array's elements in position order. The +/// profile's numbering of a row's nodes, and the order of a needle's pairs; +/// not the stored order ([`StoredOrder`]). pub fn flatten(doc: &Value, order: KeyOrder) -> Vec> { let mut flattener = Flattener { order, @@ -979,8 +1021,8 @@ pub fn l_store(column: &Column, ids: &mut Ids, docs: &[Value]) -> Vec], at: usize, sealed_root: bool) -> Vec { let mut inside = vec![false; nodes.len()]; inside[at] = true; @@ -1198,6 +1240,7 @@ pub fn leak(column: &Column, clauses: Clauses, docs: &[Value], queries: &[Query] .collect(); Leakage { key_tag_len: KEY_TAG_LEN, + stored_order: StoredOrder::Random, rows, queries, ordering_ranks: ids.ordering_ranks(), diff --git a/packages/stack-encrypt/tests/json_leakage/simulator.rs b/packages/stack-encrypt/tests/json_leakage/simulator.rs index 5a449fd81..713c8b9ba 100644 --- a/packages/stack-encrypt/tests/json_leakage/simulator.rs +++ b/packages/stack-encrypt/tests/json_leakage/simulator.rs @@ -26,6 +26,14 @@ //! self tag from its own token, by the rule the profile states for it. //! Every other parent and self tag is random bytes, as a tag under a token //! the database never sees is to it. +//! +//! The stored order is drawn as the crate draws it +//! ([`StoredOrder::Random`]): per row, the root at slot `0` and the other +//! nodes at a uniformly random permutation of the slots `1..n`, and every +//! list of the stored `q` shuffled, each by Fisher–Yates over the +//! simulator's generator. The profile names nodes, not slots, so the tags +//! the chain programs follow each node to wherever it is stored. A needle is +//! sent in the walk's order, unshuffled, as the client sends it. use std::collections::{BTreeMap, HashMap}; @@ -41,7 +49,7 @@ use super::chain::{ use super::observe::QueryKind; use super::profile::{ GroupLeakage, KindOrder, Leakage, OrderKind, OrderingLeak, Placement, QueryLeakage, RowLeakage, - Step, TokenName, TokenStep, + Step, StoredOrder, TokenName, TokenStep, }; /// The version byte `vitaminc-encrypt` writes at the front of every @@ -81,6 +89,15 @@ fn b64(bytes: &[u8]) -> String { Base64::encode_string(bytes) } +/// A uniform shuffle of `items` (Fisher–Yates). Each draw reduces a 64-bit +/// word modulo at most a few hundred, a bias below `2^-55`. +fn shuffle(items: &mut [T], rng: &mut TestRng) { + for i in (1..items.len()).rev() { + let j = (rng.next_u64() % (i as u64 + 1)) as usize; + items.swap(i, j); + } +} + /// The tags a row's entries carry where some query's token links them. #[derive(Default)] struct Linked { @@ -315,8 +332,11 @@ impl Simulator<'_> { bytes.clone() } - fn group(&mut self, group: &GroupLeakage) -> Value { - let pairs: Vec = group + /// A projection's wire form: in the walk's order, as a needle is sent, + /// or with every list shuffled, as a stored `q` is under + /// [`StoredOrder::Random`]. + fn group(&mut self, group: &GroupLeakage, shuffled: bool) -> Value { + let mut pairs: Vec = group .pairs .iter() .map(|&(s, vs)| { @@ -325,10 +345,32 @@ impl Simulator<'_> { json!({"s": b64(&s), "vs": b64(&vs)}) }) .collect(); - let elements: Vec = group.elements.iter().map(|g| self.group(g)).collect(); + let mut elements: Vec = group + .elements + .iter() + .map(|g| self.group(g, shuffled)) + .collect(); + if shuffled { + shuffle(&mut pairs, self.rng); + shuffle(&mut elements, self.rng); + } json!({"sv": pairs, "el": elements}) } + /// The slot each of a row's `n` nodes is stored at, by its index in the + /// profile's numbering: the root at `0` and the rest at a uniformly + /// random permutation of `1..n`, or the numbering itself where the + /// profile states the depth-first order. + fn slots(&mut self, n: usize) -> Vec { + let mut slots: Vec = (0..n).collect(); + if self.leakage.stored_order == StoredOrder::Random { + if let Some(rest) = slots.get_mut(1..) { + shuffle(rest, self.rng); + } + } + slots + } + /// Entry `i`'s ordinal under its parent, whose token is `parent`: an /// element's position, or a member's rank among its sibling members /// under the simulator's own keyed rank. @@ -406,8 +448,8 @@ impl Simulator<'_> { let _ = linked[r].x.insert(h, self_tag(&iv, &t)); let _ = tokens.insert(h, t); } - // Parents before children: the stored order puts them first, - // but a mutant may not, so by depth. + // Parents before children: the profile's numbering puts them + // first, but a mutant's may not, so by depth. let mut reach = read.reach[r].clone(); reach.sort_by_key(|&i| (depth(row, i), i)); for c in reach { @@ -454,7 +496,7 @@ impl Simulator<'_> { header.extend_from_slice(&u16::try_from(tag.len()).expect("a short tag").to_le_bytes()); header.extend_from_slice(&tag); - let entries: Vec = row + let by_node: Vec = row .entries .iter() .enumerate() @@ -483,7 +525,15 @@ impl Simulator<'_> { wire }) .collect(); - let q = self.group(&row.projection); + let slots = self.slots(by_node.len()); + let mut entries = vec![Value::Null; by_node.len()]; + for (wire, slot) in by_node.into_iter().zip(slots) { + entries[slot] = wire; + } + let q = self.group( + &row.projection, + self.leakage.stored_order == StoredOrder::Random, + ); json!({"h": b64(&header), "sv": entries, "q": q}) } @@ -500,13 +550,13 @@ impl Simulator<'_> { (QueryKind::Subtree, json!({"t": tokens, "budget": budget})) } QueryLeakage::Contains { needle, alt } => { - let mut wire = self.group(needle); + let mut wire = self.group(needle, false); if let Some(alt) = alt { - wire["alt"] = self.group(alt); + wire["alt"] = self.group(alt, false); } (QueryKind::Contains, wire) } - QueryLeakage::Equals { needle } => (QueryKind::Equals, self.group(needle)), + QueryLeakage::Equals { needle } => (QueryKind::Equals, self.group(needle, false)), QueryLeakage::Order { selector, bound } => { let s = self.selector(*selector); let op = self.ordering(bound.class, bound.len); diff --git a/packages/stack-encrypt/tests/json_properties.rs b/packages/stack-encrypt/tests/json_properties.rs index c37d41749..421a34f8f 100644 --- a/packages/stack-encrypt/tests/json_properties.rs +++ b/packages/stack-encrypt/tests/json_properties.rs @@ -15,7 +15,11 @@ //! the concrete paths its path names, below the bound, and the probes, and //! no token of a node above them. //! - **What a row stores.** `x` on every entry, `p` on every entry but the -//! root and the heads, under either root option. +//! root and the heads, under either root option; the entries in a random +//! order that is a permutation of the nodes with the root at slot 0. +//! - **Placement and tampering.** The rows of a read open to the same value +//! in any order they arrive in; and any one row changed, dropped, +//! repeated or misreported is refused, with the error that names how. mod common; @@ -23,6 +27,7 @@ use std::collections::BTreeSet; use common::stack_cipher; use proptest::prelude::*; +use proptest::test_runner::TestCaseError; use serde_json::{json, Map, Value}; use stack_encrypt::json::{ JsonDocument, JsonExtracted, JsonNode, JsonOptions, JsonPath, JsonSubtree, Segment, @@ -109,8 +114,9 @@ async fn read( .unwrap_or_else(|error| panic!("{path}: {error}")) } -/// Every node's path in `value`, in the order the index stores the nodes: -/// depth first, the root first, members in key order. +/// Every node's path in `value`, depth first, the root first, members in +/// key order. The index stores the nodes in a random order instead; a node's +/// slot is found by its path's token ([`slot`]). fn node_paths(value: &Value) -> Vec { fn visit(value: &Value, at: &JsonPath, out: &mut Vec) { out.push(at.clone()); @@ -133,6 +139,31 @@ fn node_paths(value: &Value) -> Vec { out } +/// The slot the node at the concrete `path` is stored at, found by the +/// self tag of the path's token, as the server finds it. +async fn slot( + cipher: &Cipher, + doc: &JsonDocument, + path: &JsonPath, + options: &JsonOptions, +) -> Option { + let k = key(cipher, path, options, 0).await; + doc.find(&k.token_bytes()[0]).map(|(j, _)| j) +} + +/// `items` in the order a Fisher–Yates shuffle driven by `seed` puts them: +/// any permutation, reproducibly from the case. +fn permuted(mut items: Vec, seed: u64) -> Vec { + let mut state = seed | 1; + for i in (1..items.len()).rev() { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + items.swap(i, (state % (i as u64 + 1)) as usize); + } + items +} + fn append(at: &JsonPath, step: &Segment) -> JsonPath { let text = match step { Segment::Key(key) => format!("[{}]", serde_json::to_string(key).unwrap()), @@ -313,9 +344,9 @@ proptest! { } /// Two rows of one shape and one set of keys, with different values and - /// different data keys: the same selectors in the same order, the same - /// nodes found by every path's token, and no self tag, parent tag or - /// ciphertext in common. + /// different data keys: the same selectors (each in its own random + /// order), every path's token finding the node at that path in both, and + /// no self tag, parent tag or ciphertext in common. #[test] fn selectors_and_tokens_follow_the_shape_alone(value in document(), sealed in any::()) { let rt = runtime(); @@ -325,15 +356,21 @@ proptest! { let one = seal(&cipher, &value, &options).await; let two = seal(&cipher, &revalue(&value), &options).await; prop_assert_ne!(one.header().iv(), two.header().iv()); - let s = |d: &JsonDocument| d.entries().iter().map(|e| *e.selector()).collect::>(); - prop_assert_eq!(s(&one), s(&two), "the same selectors, in the same order"); + let s = |d: &JsonDocument| { + let mut s = d.entries().iter().map(|e| *e.selector()).collect::>(); + s.sort_unstable(); + s + }; + prop_assert_eq!(s(&one), s(&two), "the same multiset of selectors"); for path in node_paths(&value) { - let k = key(&cipher, &path, &options, 4).await; - prop_assert_eq!( - one.find(&k.token_bytes()[0]).map(|(j, _)| j), - two.find(&k.token_bytes()[0]).map(|(j, _)| j), - "{} at one position in both rows", path + let (a, b) = ( + slot(&cipher, &one, &path, &options).await, + slot(&cipher, &two, &path, &options).await, ); + let (Some(a), Some(b)) = (a, b) else { + return Err(TestCaseError::fail(format!("{path} found in both rows"))); + }; + prop_assert_eq!(one.entries()[a].selector(), two.entries()[b].selector(), "{}", path); } let mut tags = BTreeSet::new(); let mut total = 0; @@ -353,19 +390,130 @@ proptest! { })?; } - /// `x` on every entry, `p` on every entry but the root and the heads. + /// `x` on every entry, `p` on every entry but the root and the heads; + /// and the entries are a permutation of the nodes, the root at slot 0. #[test] fn every_entry_carries_a_self_tag_and_only_the_heads_lack_a_parent_tag(value in document(), sealed in any::()) { let rt = runtime(); rt.block_on(async { let cipher = stack_cipher().await; - let doc = seal(&cipher, &value, &options(sealed)).await; + let options = options(sealed); + let doc = seal(&cipher, &value, &options).await; let wire = serde_json::to_value(&doc).unwrap(); - for (j, (entry, path)) in wire["sv"].as_array().unwrap().iter().zip(node_paths(&value)).enumerate() { + let sv = wire["sv"].as_array().unwrap(); + let paths = node_paths(&value); + prop_assert_eq!(sv.len(), paths.len(), "one entry per node"); + let mut slots = Vec::new(); + for path in &paths { + let Some(j) = slot(&cipher, &doc, path, &options).await else { + return Err(TestCaseError::fail(format!("{path} is stored"))); + }; + slots.push(j); + let entry = &sv[j]; prop_assert!(entry.get("x").is_some(), "entry {} has x", j); let head = path.segments().is_empty() || (sealed && path.segments().len() == 1); prop_assert_eq!(entry.get("p").is_none(), head, "entry {} ({}): p absent only on a head", j, path); } + prop_assert_eq!(slots[0], 0, "the root at slot 0"); + slots.sort_unstable(); + prop_assert_eq!(slots, (0..paths.len()).collect::>(), "a permutation of the slots"); + Ok(()) + })?; + } + + /// The rows of a read open to the same value whatever order they arrive + /// in: the client places them by the chain, not by their slots or their + /// order. + #[test] + fn a_reads_rows_open_in_any_order(value in document(), sealed in any::(), pick in any::(), seed in any::()) { + let rt = runtime(); + rt.block_on(async { + let cipher = stack_cipher().await; + let options = options(sealed); + let doc = seal(&cipher, &value, &options).await; + let paths = node_paths(&value); + let path = &paths[pick.index(paths.len())]; + let k = key(&cipher, path, &options, 4).await; + let shuffled = permuted(rows(&doc, &k), seed); + let opened = cipher + .decrypt_as::, JsonSubtree>(k.open(shuffled), aead()) + .await + .unwrap_or_else(|error| panic!("{path}: {error}")); + prop_assert_eq!(opened, path.select(&value).first().cloned().cloned(), "{}", path); + Ok(()) + })?; + } + + /// Any one row of the read of the whole document (the root open, so the + /// read walks every entry) changed, dropped, repeated or misreported is + /// refused, with the refusal that says how: a changed ciphertext or a + /// dropped child by the cipher, since a parent's ciphertext commits to + /// its children's tags; a dropped root by the rows no read reaches; a + /// repeated row before any key is retrieved; a wrong depth by the depth + /// check. + #[test] + fn any_one_tampered_row_is_refused(value in document(), pick in any::(), how in 0u8..4, byte in any::()) { + let rt = runtime(); + rt.block_on(async { + let cipher = stack_cipher().await; + let options = options(false); + let doc = seal(&cipher, &value, &options).await; + let k = key(&cipher, &JsonPath::root(), &options, 4).await; + let mut rows = rows(&doc, &k); + let at = pick.index(rows.len()); + let top = at == 0; + let mut wire = serde_json::to_value(&rows[at]).unwrap(); + let expected = match how { + 0 => { + // One byte of the ciphertext, the tag's included, changed. + let c = wire["c"].as_str().unwrap().to_owned(); + let mut bytes = c.into_bytes(); + // Lowered to base85's zero digit, so the group it sits + // in still decodes: the first character from a random + // place on that is not zero already. + let start = byte.index(bytes.len()); + let i = (0..bytes.len()) + .map(|k| (start + k) % bytes.len()) + .find(|&i| bytes[i] != b'0') + .unwrap(); + bytes[i] = b'0'; + wire["c"] = json!(String::from_utf8(bytes).unwrap()); + rows[at] = serde_json::from_value(wire).unwrap(); + "aead" + } + 1 => { + let _ = rows.remove(at); + if top && rows.is_empty() { + // Nothing returned: the database's word that the + // row has no node there (ADR 0010). + "none" + } else if top { + "rows the walked paths reach" + } else { + "aead" + } + } + 2 => { + let copy = rows[at].clone(); + rows.push(copy); + "rows of distinct entries" + } + _ => { + wire["d"] = json!(wire["d"].as_u64().unwrap() + 1); + rows[at] = serde_json::from_value(wire).unwrap(); + "a row at the depth the chain gives it" + } + }; + let opened = cipher + .decrypt_as::, JsonSubtree>(k.open(rows), aead()) + .await; + match (expected, opened) { + ("aead", Err(Error::Aead)) | ("none", Ok(None)) => {} + (expected, Err(Error::JsonWrongEntry { expected: got })) if got == expected => {} + (expected, got) => { + return Err(TestCaseError::fail(format!("row {at}, tamper {how}: expected {expected}, got {got:?}"))); + } + } Ok(()) })?; } @@ -461,15 +609,19 @@ proptest! { .iter() .map(|t| doc.find(t).map(|(j, _)| j).expect("every token names a node of the witness")) .collect(); - let at: Vec = expected - .iter() - .map(|(q, _)| paths.iter().position(|p| p == q).unwrap()) - .collect(); + // Where each node of the witness is stored, by its own path's + // key: the slots are random, so the read's tokens are checked + // against the nodes the paths name, not against positions. + let mut stored = std::collections::BTreeMap::new(); + for p in &paths { + let _ = stored.insert(p.clone(), slot(&cipher, &doc, p, &options).await.unwrap()); + } + let at: Vec = expected.iter().map(|(q, _)| stored[q]).collect(); prop_assert_eq!(&found, &at, "each token finds the node at its path"); for (q, _) in &expected { for len in 0..q.segments().len() { let ancestor = q.segments()[..len].iter().fold(JsonPath::root(), |a, s| append(&a, s)); - let j = paths.iter().position(|p| *p == ancestor).unwrap(); + let j = stored[&ancestor]; prop_assert!(!found.contains(&j), "no token of {}, above {}", ancestor, q); } } @@ -555,3 +707,26 @@ async fn the_self_tag_costs_about_a_tenth_of_the_stored_size() { ); } } + +/// The stored order is drawn afresh for every document: ten seals of one +/// ten-node document do not all store it alike, while each keeps the root +/// at slot 0. (Two seals agree with probability 1/9!, so ten all agreeing +/// is never seen.) +#[tokio::test] +async fn every_seal_draws_its_own_order() { + let cipher = stack_cipher().await; + let value = + json!({"hello": "w", "nested": {"deep": "c", "list": [{"x": 1}, {"x": 2}]}, "number": 1}); + let options = options(true); + let mut orders = BTreeSet::new(); + for _ in 0..10 { + let doc = seal(&cipher, &value, &options).await; + let mut order = Vec::new(); + for path in node_paths(&value) { + order.push(slot(&cipher, &doc, &path, &options).await.unwrap()); + } + assert_eq!(order[0], 0, "the root at slot 0"); + let _ = orders.insert(order); + } + assert!(orders.len() > 1, "ten seals, one order: {orders:?}"); +} diff --git a/packages/stack-encrypt/tests/json_read_checks.rs b/packages/stack-encrypt/tests/json_read_checks.rs index 3c8f1fc85..de777ddf0 100644 --- a/packages/stack-encrypt/tests/json_read_checks.rs +++ b/packages/stack-encrypt/tests/json_read_checks.rs @@ -1,15 +1,20 @@ //! What the client checks when it opens the rows a JSON index read //! returned, with no database: the entry `->` returns must carry the -//! selector the client derived for the path, and every row of a subtree -//! read must sit where the key that keyed the read says its rows are. -//! Each entry's ciphertext is bound to its own selector and position, so -//! without these checks any entry of the row would open in place of the one -//! asked for. +//! selector the client derived for the path, and the rows of a subtree read +//! must be the subtree of the node the key's token names. The cascade +//! refuses, at the cipher ([`Error::Aead`]), any change to the rows beneath +//! that node: a row swapped, dropped, added, reordered or moved under +//! another parent changes a tag on the way up. The client's own checks +//! ([`Error::JsonWrongEntry`]) refuse the rest: a repeated row, a row no +//! read reaches, a reported depth the chain does not give, a self tag no +//! rule derives, and a node at the top of a read whose selector or label is +//! not its path's. //! //! The rows here are built as the SQL returns them (the entry, the header -//! and the ordinal grafted, and a walk's depth), from a sealed document, so -//! every entry in them is genuine. What varies is which entries a dishonest -//! database hands back. +//! and the slot grafted, and a walk's depth), from a sealed document, so +//! every entry in them is genuine unless a test rewrites a tag. Entries are +//! stored in a random order, so a test finds a node's row by its path +//! ([`slot`]), never by a fixed slot number. mod common; @@ -41,8 +46,8 @@ fn aead() -> AeadContext { AeadContext::from(nonempty!("users").with("profile")) } -/// `$`, `hello` (1), `nested` (2), `deep` (3), `list` (4), its elements (5, -/// 7) and their `x` (6, 8), `number` (9): depth-first, members in key order. +/// `$`, `hello`, `nested`, `nested.deep`, `nested.list`, its two elements +/// and their `x`, and `number`: ten nodes, stored in a random order. fn fixture() -> Value { json!({"hello": "world-1", "nested": {"deep": "constant", "list": [{"x": 1}, {"x": 2}]}, "number": 1}) } @@ -107,6 +112,24 @@ fn open_root() -> JsonOptions { } } +/// The slot the node at the concrete `path` is stored at, found as the +/// server finds it: by the self tag of the path's token. +async fn slot(cipher: &Cipher, doc: &JsonDocument, path: &str) -> usize { + slot_with(cipher, doc, path, &JsonOptions::default()).await +} + +async fn slot_with( + cipher: &Cipher, + doc: &JsonDocument, + path: &str, + options: &JsonOptions, +) -> usize { + let key = key_with(cipher, path, options).await; + doc.find(&key.token_bytes()[0]) + .unwrap_or_else(|| panic!("the document has {path}")) + .0 +} + /// The entry at `j` as an extractor returns it, at depth `d` when a walk /// returned it. fn row(doc: &JsonDocument, j: usize, d: Option) -> JsonNode { @@ -160,6 +183,15 @@ mod assertions { other => panic!("expected JsonWrongEntry ({expected}), got {other:?}"), } } + + /// The read was refused by the cipher: some node did not open under the + /// digest of the children the chain gave it, or under its own AAD. + pub(crate) fn aead(result: Result) { + assert!( + matches!(result, Err(Error::Aead)), + "expected Aead, got {result:?}" + ); + } } mod given_the_entry_arrow_returned { @@ -170,7 +202,8 @@ mod given_the_entry_arrow_returned { let (cipher, _) = cipher().await; let doc = seal(&cipher, &fixture()).await; let hello = selector(&cipher, "$.hello").await; - let opened = open_extracted(&cipher, hello.open(row(&doc, 1, None))).await; + let at = slot(&cipher, &doc, "$.hello").await; + let opened = open_extracted(&cipher, hello.open(row(&doc, at, None))).await; assert_eq!(opened.unwrap(), json!("world-1"), "the value at the path"); } @@ -181,9 +214,14 @@ mod given_the_entry_arrow_returned { let hello = selector(&cipher, "$.hello").await; // `$.number` is a genuine entry of the row, and opens on its own // selector: the database answered `$.hello` with it. - let extracted = hello.open(row(&doc, 9, None)); + let number = slot(&cipher, &doc, "$.number").await; + let extracted = hello.open(row(&doc, number, None)); assert_eq!(extracted.selector(), hello, "the selector asked for"); - assert_eq!(extracted.node().ordinal(), 9, "the row as returned"); + assert_eq!( + extracted.node().ordinal(), + number as u64, + "the row as returned" + ); assert_wrong(open_extracted(&cipher, extracted).await); assert_eq!( retrieved.load(Ordering::SeqCst), @@ -218,7 +256,8 @@ mod given_the_entry_arrow_returned { let hello = selector(&cipher, "$.hello").await; // Passing the check by rewriting `s` breaks the ciphertext's // binding to its own selector instead. - let mut wire = serde_json::to_value(doc.node(9).unwrap()).unwrap(); + let number = slot(&cipher, &doc, "$.number").await; + let mut wire = serde_json::to_value(doc.node(number).unwrap()).unwrap(); wire["s"] = json!(hello.to_string()); let relabelled: JsonNode = serde_json::from_value(wire).unwrap(); let opened = open_extracted(&cipher, hello.open(relabelled)).await; @@ -230,12 +269,33 @@ mod given_the_entry_arrow_returned { let (cipher, _) = cipher().await; let doc = seal(&cipher, &fixture()).await; let second = selector(&cipher, "$.nested.list[1].x").await; - // `->` returns the first element at the path; the selector cannot - // say which position was meant. - let opened = open_extracted(&cipher, second.open(row(&doc, 6, None))).await; + // `->` returns whichever element at the path the stored order puts + // first; the selector cannot say which position was meant. + let first = slot(&cipher, &doc, "$.nested.list[0].x").await; + let opened = open_extracted(&cipher, second.open(row(&doc, first, None))).await; assert_eq!(opened.unwrap(), json!(1), "the first element's x"); } + /// A container's entry is sealed over its children's tags, which `->` + /// does not return: alone it does not open, and the cipher refuses it. + /// An empty container opens, and holds no value of its own. + #[tokio::test] + async fn a_container_does_not_open_alone() { + let (cipher, _) = cipher().await; + let doc = seal(&cipher, &json!({"full": {"a": 1}, "empty": {}})).await; + let full = selector(&cipher, "$.full").await; + let at = slot(&cipher, &doc, "$.full").await; + let opened = open_extracted(&cipher, full.open(row(&doc, at, None))).await; + assert!(matches!(opened, Err(Error::Aead)), "{opened:?}"); + let empty = selector(&cipher, "$.empty").await; + let at = slot(&cipher, &doc, "$.empty").await; + let opened = open_extracted(&cipher, empty.open(row(&doc, at, None))).await; + assert!( + matches!(opened, Err(Error::OpenedType { .. })), + "{opened:?}" + ); + } + #[tokio::test] async fn the_same_path_from_another_row_opens_by_design() { let (cipher, _) = cipher().await; @@ -243,7 +303,8 @@ mod given_the_entry_arrow_returned { let hello = selector(&cipher, "$.hello").await; // Row provenance is the database's word (ADR 0010): an entry // returned with its own row's header opens. - let opened = open_extracted(&cipher, hello.open(row(&other, 1, None))).await; + let at = slot(&cipher, &other, "$.hello").await; + let opened = open_extracted(&cipher, hello.open(row(&other, at, None))).await; assert_eq!(opened.unwrap(), json!("world-2"), "another row's value"); } @@ -256,10 +317,12 @@ mod given_the_rows_of_a_read { use super::*; const REACHED: &str = "a node the walked path reaches"; - const NEXT: &str = "the next node of the walked subtree"; + const UNREACHED: &str = "rows the walked paths reach"; + const DEPTH: &str = "a row at the depth the chain gives it"; + const RULE: &str = "a child the member or element rule names"; #[tokio::test] - async fn the_honest_rows_open_to_the_nodes_value() { + async fn the_honest_rows_open_to_the_nodes_value_in_any_order() { let (cipher, _) = cipher().await; let doc = seal(&cipher, &fixture()).await; let nested = key(&cipher, "$.nested").await; @@ -267,41 +330,63 @@ mod given_the_rows_of_a_read { assert_eq!(rows.len(), 7, "nested and its descendants"); let value = open_value(&cipher, nested.open(rows.clone())).await; assert_eq!(value.unwrap(), fixture()["nested"], "the sub-document"); - let list = open_list(&cipher, nested.open(rows)).await; + let list = open_list(&cipher, nested.open(rows.clone())).await; assert_eq!( list.unwrap(), vec![fixture()["nested"].clone()], "one node, one value" ); + // The rows are placed by the chain, not by the order they arrive in. + let mut reversed = rows.clone(); + reversed.reverse(); + let mut by_slot = rows; + by_slot.sort_by_key(JsonNode::ordinal); + for rows in [reversed, by_slot] { + let value = open_value(&cipher, nested.open(rows)).await; + assert_eq!(value.unwrap(), fixture()["nested"]); + } } + /// The database answers `$.a` with `$.b`'s subtree: genuine entries, + /// under `$.b`'s path. As they are, no row carries the self tag of the + /// token the read sent; with `$.b`'s self tag rewritten to `$.a`'s, the + /// chain finds no children of `$.a` among them, so `$.b` does not open + /// under the digest of no children; and a leaf `$.b` that does open + /// carries `$.b`'s selector, not `$.a`'s. #[tokio::test] - async fn the_rows_of_another_field_of_the_same_shape_are_refused() { + async fn the_rows_of_another_field_are_refused() { let (cipher, _) = cipher().await; let doc = seal(&cipher, &json!({"a": {"x": 1}, "b": {"x": 2}})).await; let a = key(&cipher, "$.a").await; let b = key(&cipher, "$.b").await; - // The database answers `$.a` with `$.b`'s subtree: genuine entries, - // in their places, under `$.b`'s path. - assertions::wrong_entry(open_value(&cipher, a.open(walk(&doc, &b))).await, REACHED); - // The same with `$.b`'s self tag rewritten to `$.a`'s token's: the - // self tag is the database's to write, and the selector still says - // which path the node is at. + assertions::wrong_entry(open_value(&cipher, a.open(walk(&doc, &b))).await, UNREACHED); + let forged = with_x(walk(&doc, &b), 0, x_of(&doc, &a)); + assertions::aead(open_value(&cipher, a.open(forged)).await); + + let doc = seal(&cipher, &json!({"a": 1, "b": 2})).await; let forged = with_x(walk(&doc, &b), 0, x_of(&doc, &a)); assertions::wrong_entry(open_value(&cipher, a.open(forged)).await, REACHED); } + /// The first element carries the same selector as the second, and here + /// the second's self tag: its decrypted position tells them apart. A + /// container element in its place is refused before that, by the + /// cascade, since the chain finds none of its children under the second + /// element's token. #[tokio::test] async fn another_position_of_the_array_is_refused_by_its_label() { let (cipher, _) = cipher().await; + let doc = seal(&cipher, &json!({"arr": [10, 20]})).await; + let second = key(&cipher, "$.arr[1]").await; + let first = key(&cipher, "$.arr[0]").await; + let forged = with_x(walk(&doc, &first), 0, x_of(&doc, &second)); + assertions::wrong_entry(open_value(&cipher, second.open(forged)).await, REACHED); + let doc = seal(&cipher, &fixture()).await; let second = key(&cipher, "$.nested.list[1]").await; let first = key(&cipher, "$.nested.list[0]").await; - // The first element carries the same selector as the second, and - // here the second's self tag: its decrypted position tells them - // apart. let forged = with_x(walk(&doc, &first), 0, x_of(&doc, &second)); - assertions::wrong_entry(open_value(&cipher, second.open(forged)).await, REACHED); + assertions::aead(open_value(&cipher, second.open(forged)).await); } #[tokio::test] @@ -312,7 +397,7 @@ mod given_the_rows_of_a_read { // `$.hello`'s own entry with another self tag: the database walked // from a token the read did not send. let forged = with_x(walk(&doc, &hello), 0, "AAAAAAAAAAA=".into()); - assertions::wrong_entry(open_value(&cipher, hello.open(forged)).await, REACHED); + assertions::wrong_entry(open_value(&cipher, hello.open(forged)).await, UNREACHED); } #[tokio::test] @@ -320,9 +405,9 @@ mod given_the_rows_of_a_read { let (cipher, _) = cipher().await; let doc = seal(&cipher, &fixture()).await; let xs = key(&cipher, "$.nested.list[*].x").await; - // Both x, the second carrying the first's self tag: each a genuine - // node at the selector, but the read sent each token once, and a - // row matches only a token after the one before it. + // Both x, the second carrying the first's self tag: the first + // token's read takes one of them, and the other is reached by no + // token the read sent. let rows = walk(&doc, &xs); assert_eq!(rows.len(), 2, "both x"); let first = serde_json::to_value(&rows[0]).unwrap()["x"] @@ -330,7 +415,7 @@ mod given_the_rows_of_a_read { .unwrap() .to_owned(); let forged = with_x(rows, 1, first); - assertions::wrong_entry(open_list(&cipher, xs.open(forged)).await, REACHED); + assertions::wrong_entry(open_list(&cipher, xs.open(forged)).await, UNREACHED); } #[tokio::test] @@ -338,10 +423,10 @@ mod given_the_rows_of_a_read { let (cipher, retrieved) = cipher().await; let doc = seal(&cipher, &fixture()).await; let nested = key(&cipher, "$.nested").await; - // nested's rows with its first member, `deep` (3), returned twice: - // a genuine entry at its own position, which would let the database - // line one container up as often as it likes. An honest database - // never returns one, so it is the wrong entry, not a shape. + // nested's rows with one of them returned twice: a genuine entry at + // its own slot, which would let the database line one container up + // as often as it likes. An honest database never returns one, so it + // is the wrong entry, not a shape. let mut rows = walk(&doc, &nested); rows.insert(2, rows[1].clone()); assertions::wrong_entry( @@ -361,10 +446,10 @@ mod given_the_rows_of_a_read { let doc = seal(&cipher, &fixture()).await; let other = seal(&cipher, &fixture()).await; let xs = key(&cipher, "$.nested.list[*].x").await; - // The first x (6) from this row and the second (8) from another, - // each with its own row's header. Either row's answer alone opens - // (ADR 0010), but a read is of one row, and its rows of two are not - // the answer to it. + // The first x from this row and the second from another, each with + // its own row's header. Either row's answer alone opens (ADR 0010), + // but a read is of one row, and its rows of two are not the answer + // to it. let rows = vec![walk(&doc, &xs)[0].clone(), walk(&other, &xs)[1].clone()]; assertions::wrong_entry( open_list(&cipher, xs.open(rows)).await, @@ -377,52 +462,198 @@ mod given_the_rows_of_a_read { let (cipher, _) = cipher().await; let doc = seal(&cipher, &fixture()).await; let nested = key(&cipher, "$.nested").await; - let forged = with_x(walk(&doc, &nested), 2, "AAAAAAAAAAA=".into()); - assertions::wrong_entry(open_value(&cipher, nested.open(forged)).await, NEXT); + let deep = slot(&cipher, &doc, "$.nested.deep").await; + let rows = walk(&doc, &nested); + let at = rows + .iter() + .position(|n| n.ordinal() == deep as u64) + .unwrap(); + let forged = with_x(rows, at, "AAAAAAAAAAA=".into()); + assertions::wrong_entry(open_value(&cipher, nested.open(forged)).await, RULE); } + /// A member whose self tag is rewritten to the one the element rule + /// derives at its ordinal: the server would take it for an element, and + /// so would the chain, but its decrypted label is a key. #[tokio::test] - async fn a_row_of_another_path_spliced_in_is_refused() { + async fn a_child_under_the_other_rule_is_refused_by_its_label() { + let (cipher, _) = cipher().await; + let doc = seal(&cipher, &json!({"a": {"k": 1}})).await; + let a = key(&cipher, "$.a").await; + let k = slot(&cipher, &doc, "$.a.k").await; + let s = selector_bytes(&selector(&cipher, "$.a.k").await); + let as_element = chain::element_token(&a.token_bytes()[0], &s, 0); + let rows = walk(&doc, &a); + let at = rows.iter().position(|n| n.ordinal() == k as u64).unwrap(); + let forged = with_x( + rows, + at, + base64(&chain::self_tag(doc.header().iv(), &as_element)), + ); + assertions::wrong_entry( + open_value(&cipher, a.open(forged)).await, + "a child under the rule its label names", + ); + } + + #[tokio::test] + async fn a_row_of_another_path_added_to_the_answer_is_refused() { let (cipher, _) = cipher().await; let doc = seal(&cipher, &fixture()).await; let nested = key(&cipher, "$.nested").await; - // `number` (9) sits right after nested's subtree in the document, - // so it follows the rows without a gap; no token the read sent - // names it. - let mut rows = walk(&doc, &nested); - rows.push(row(&doc, 9, Some(1))); - assertions::wrong_entry(open_value(&cipher, nested.open(rows)).await, REACHED); + // `number`, a genuine entry no token the read sent reaches. + let number = slot(&cipher, &doc, "$.number").await; + for d in [Some(1), None] { + let mut rows = walk(&doc, &nested); + rows.push(row(&doc, number, d)); + assertions::wrong_entry(open_value(&cipher, nested.open(rows)).await, UNREACHED); + } + } + + /// `number`, a genuine entry, with its parent tag and self tag rewritten + /// to make it a third element of `nested.list`: the chain takes it for + /// one, and the list, sealed over two, does not open. + #[tokio::test] + async fn an_added_child_is_refused_by_the_cascade() { + let (cipher, _) = cipher().await; + let doc = seal(&cipher, &fixture()).await; + let nested = key(&cipher, "$.nested").await; + let list = key(&cipher, "$.nested.list").await; + // The server derives a hit's token from the hit's own selector. + let s = selector_bytes(&selector(&cipher, "$.number").await); + let iv = doc.header().iv(); + let third = chain::element_token(&list.token_bytes()[0], &s, 2); + let number = slot(&cipher, &doc, "$.number").await; + let mut wire = serde_json::to_value(row(&doc, number, Some(2))).unwrap(); + wire["x"] = json!(base64(&chain::self_tag(iv, &third))); + wire["p"] = json!(base64(&chain::parent_tag(iv, &list.token_bytes()[0], 2))); let mut rows = walk(&doc, &nested); - rows.push(row(&doc, 9, None)); - assertions::wrong_entry(open_value(&cipher, nested.open(rows)).await, REACHED); + rows.push(serde_json::from_value(wire).unwrap()); + assertions::aead(open_value(&cipher, nested.open(rows)).await); } #[tokio::test] - async fn a_dropped_row_is_refused() { + async fn a_dropped_child_is_refused_by_the_cascade() { let (cipher, _) = cipher().await; let doc = seal(&cipher, &fixture()).await; let nested = key(&cipher, "$.nested").await; - // The first element (5) is dropped, so its `x` (6) would be taken - // for the element itself. - let rows: Vec = walk(&doc, &nested) - .into_iter() - .filter(|n| n.ordinal() != 5) - .collect(); - assertions::wrong_entry(open_value(&cipher, nested.open(rows)).await, NEXT); + // The first element is dropped: the list's children are then + // probed from ordinal 0 and none is found, and its x is reached by + // nothing. + let first = slot(&cipher, &doc, "$.nested.list[0]").await; + let rows = without(walk(&doc, &nested), first); + assertions::aead(open_value(&cipher, nested.open(rows)).await); + } + + #[tokio::test] + async fn a_truncated_subtree_is_refused_by_the_cascade() { + let (cipher, _) = cipher().await; + let doc = seal(&cipher, &fixture()).await; + let nested = key(&cipher, "$.nested").await; + // The deepest node of the last element left out: the element says + // nothing of it in a position, but its ciphertext commits to its + // child's tag. + let last = slot(&cipher, &doc, "$.nested.list[1].x").await; + let rows = without(walk(&doc, &nested), last); + assertions::aead(open_value(&cipher, nested.open(rows)).await); + } + + #[tokio::test] + async fn a_gap_between_two_members_is_refused_by_the_cascade() { + let (cipher, _) = cipher().await; + let doc = seal(&cipher, &json!({"a": {"x": 1, "y": {"z": 2}, "w": 3}})).await; + let a = key(&cipher, "$.a").await; + // The database leaves `x` out; `w` and `y` are each genuine members. + let x = slot(&cipher, &doc, "$.a.x").await; + let rows = without(walk(&doc, &a), x); + assertions::aead(open_value(&cipher, a.open(rows)).await); + } + + /// The two elements of `nested.list` with their parent tags and self + /// tags exchanged: the chain finds each at the other's ordinal, under + /// the rule its self tag now names, and the list, sealed over its + /// elements in order, does not open. + #[tokio::test] + async fn reordered_children_are_refused_by_the_cascade() { + let (cipher, _) = cipher().await; + let doc = seal(&cipher, &fixture()).await; + let list = key(&cipher, "$.nested.list").await; + let first = slot(&cipher, &doc, "$.nested.list[0]").await; + let second = slot(&cipher, &doc, "$.nested.list[1]").await; + let rows = exchange_tags(walk(&doc, &list), first, second); + assertions::aead(open_value(&cipher, list.open(rows)).await); + } + + /// The design's substitution (decision 8): a read of `$.nested.list[0]` + /// answered with the element itself and then the other element's `x` in + /// place of its own, carrying its own `x`'s self tag and parent tag. + /// Every element's `x` shares one selector, the label is `x` either way, + /// the depth and the parent's child count agree, and the forged tags + /// match; the element's ciphertext, which commits to its own `x`'s tag, + /// is the one thing that refuses it. Opened, it would read `{"x": 2}` + /// for `{"x": 1}`. + #[tokio::test] + async fn another_elements_child_in_place_of_the_nodes_own_is_refused_by_the_cascade() { + let (cipher, _) = cipher().await; + let doc = seal(&cipher, &fixture()).await; + let first = key(&cipher, "$.nested.list[0]").await; + let element = slot(&cipher, &doc, "$.nested.list[0]").await; + let own = slot(&cipher, &doc, "$.nested.list[0].x").await; + let other = slot(&cipher, &doc, "$.nested.list[1].x").await; + let honest = walk(&doc, &first); + assert_eq!( + honest.iter().map(JsonNode::ordinal).collect::>(), + [element as u64, own as u64], + "the element and its x" + ); + assert_eq!( + open_value(&cipher, first.open(honest)).await.unwrap(), + json!({"x": 1}) + ); + let swapped = vec![ + row(&doc, element, Some(0)), + with_tags_of(&doc, row(&doc, other, Some(1)), own), + ]; + assertions::aead(open_value(&cipher, first.open(swapped)).await); + } + + /// The `x` of the two elements, each moved under the other's element: + /// each element's ciphertext commits to its own child's tag. + #[tokio::test] + async fn a_child_moved_under_another_parent_is_refused_by_the_cascade() { + let (cipher, _) = cipher().await; + let doc = seal(&cipher, &fixture()).await; + let list = key(&cipher, "$.nested.list").await; + let own = slot(&cipher, &doc, "$.nested.list[0].x").await; + let other = slot(&cipher, &doc, "$.nested.list[1].x").await; + let rows = exchange_tags(walk(&doc, &list), own, other); + assertions::aead(open_value(&cipher, list.open(rows)).await); } + /// Two rows of the subtree with their ciphertexts exchanged: each now + /// opens under the other's selector and slot. #[tokio::test] - async fn a_truncated_read_is_refused() { + async fn swapped_entries_are_refused_by_the_cipher() { let (cipher, _) = cipher().await; let doc = seal(&cipher, &fixture()).await; let nested = key(&cipher, "$.nested").await; + let deep = slot(&cipher, &doc, "$.nested.deep").await; + let x = slot(&cipher, &doc, "$.nested.list[0].x").await; let mut rows = walk(&doc, &nested); - // The last row is the second element's `x` (8): without it the - // element says it has a member the rows do not hold, the same - // dropped row as a gap, at the end of the rows. - assert_eq!(rows.pop().map(|n| n.ordinal()), Some(8), "x, the last row"); - let refused = open_value(&cipher, nested.open(rows)).await; - assertions::wrong_entry(refused, NEXT); + let (i, k) = ( + rows.iter() + .position(|n| n.ordinal() == deep as u64) + .unwrap(), + rows.iter().position(|n| n.ordinal() == x as u64).unwrap(), + ); + let mut a = serde_json::to_value(&rows[i]).unwrap(); + let mut b = serde_json::to_value(&rows[k]).unwrap(); + let c = a["c"].take(); + a["c"] = b["c"].take(); + b["c"] = c; + rows[i] = serde_json::from_value(a).unwrap(); + rows[k] = serde_json::from_value(b).unwrap(); + assertions::aead(open_value(&cipher, nested.open(rows)).await); } #[tokio::test] @@ -430,7 +661,12 @@ mod given_the_rows_of_a_read { let (cipher, _) = cipher().await; let doc = seal(&cipher, &fixture()).await; let nested = key(&cipher, "$.nested").await; - for (j, d, expected) in [(6, 1, NEXT), (3, 2, NEXT), (2, 1, REACHED)] { + for (path, d) in [ + ("$.nested.list[0].x", 1), + ("$.nested.deep", 2), + ("$.nested", 1), + ] { + let j = slot(&cipher, &doc, path).await; let rows: Vec = walk(&doc, &nested) .into_iter() .map(|n| { @@ -441,7 +677,7 @@ mod given_the_rows_of_a_read { } }) .collect(); - assertions::wrong_entry(open_value(&cipher, nested.open(rows)).await, expected); + assertions::wrong_entry(open_value(&cipher, nested.open(rows)).await, DEPTH); } } @@ -450,69 +686,61 @@ mod given_the_rows_of_a_read { let (cipher, _) = cipher().await; let doc = seal(&cipher, &fixture()).await; let nested = key(&cipher, "$.nested").await; - // nested, `deep` (3) and then the second element's `x` (8) alone: - // its parent and everything between are missing. + // nested, `deep` and then the second element's `x` alone: its parent + // and everything between are missing. let rows = vec![ - row(&doc, 2, Some(0)), - row(&doc, 3, Some(1)), - row(&doc, 8, Some(3)), + row(&doc, slot(&cipher, &doc, "$.nested").await, Some(0)), + row(&doc, slot(&cipher, &doc, "$.nested.deep").await, Some(1)), + row( + &doc, + slot(&cipher, &doc, "$.nested.list[1].x").await, + Some(3), + ), ]; - assertions::wrong_entry(open_value(&cipher, nested.open(rows)).await, NEXT); + assertions::aead(open_value(&cipher, nested.open(rows)).await); } #[tokio::test] - async fn a_gap_between_two_members_is_refused() { - let (cipher, _) = cipher().await; - let doc = seal(&cipher, &json!({"a": {"x": 1, "y": {"z": 2}, "w": 3}})).await; - let a = key(&cipher, "$.a").await; - // `$.a`'s members are w (2), x (3) and y (4) with z (5) below: the - // database leaves x out, and w and y are each genuine members. - let rows: Vec = walk(&doc, &a) - .into_iter() - .filter(|n| n.ordinal() != 3) - .collect(); - assertions::wrong_entry(open_value(&cipher, a.open(rows)).await, NEXT); - } - - /// The one substitution only the consecutive-position rule catches - /// (design decision 8): a read of `$.nested.list[0]` answered with the - /// element itself (5) and then the other element's `x` (8) in place of - /// its own (6), carrying 6's self tag. Every element's `x` shares one - /// selector, the label is `x` either way, the depth and the parent's - /// child count agree, and the forged self tag matches; the row's - /// authenticated position, 8 where 6 is due, is the one thing that - /// refuses it. Opened, it would read `{"x": 2}` for `{"x": 1}`. - #[tokio::test] - async fn another_elements_child_in_place_of_the_nodes_own_is_refused_by_its_position() { + async fn the_sealed_root_among_read_rows_is_refused() { let (cipher, _) = cipher().await; let doc = seal(&cipher, &fixture()).await; - let first = key(&cipher, "$.nested.list[0]").await; - let honest = walk(&doc, &first); + let list = key(&cipher, "$.nested.list").await; + // Added to the rows, no token reaches it. + let mut rows = walk(&doc, &list); + rows.push(row(&doc, 0, Some(1))); + assertions::wrong_entry(open_value(&cipher, list.open(rows)).await, UNREACHED); + // In place of `$.hello`, under `$.hello`'s self tag: it opens, since + // it commits to no children, and is the whole document. (Alone at + // depth 0 it is the budget fallback, which opens whole and is read + // at the key's paths; with no depth it is a row of a walk.) + let hello = key(&cipher, "$.hello").await; + let fallback = with_x(vec![row(&doc, 0, Some(0))], 0, x_of(&doc, &hello)); assert_eq!( - honest.iter().map(|n| n.ordinal()).collect::>(), - [5, 6], - "the element and its x" + open_value(&cipher, hello.open(fallback)).await.unwrap(), + json!("world-1"), + "the fallback, read at the path" ); - assert_eq!( - open_value(&cipher, first.open(honest)).await.unwrap(), - json!({"x": 1}) + let forged = with_x(vec![row(&doc, 0, None)], 0, x_of(&doc, &hello)); + assertions::wrong_entry( + open_value(&cipher, hello.open(forged)).await, + "a node of the walked subtree", ); - let own_x = serde_json::to_value(doc.node(6).unwrap()).unwrap()["x"] - .as_str() - .unwrap() - .to_owned(); - let swapped = with_x(vec![row(&doc, 5, Some(0)), row(&doc, 8, Some(1))], 1, own_x); - assertions::wrong_entry(open_value(&cipher, first.open(swapped)).await, NEXT); } + /// The rows opened under another column's context: every entry's AAD + /// binds the column, and the cipher refuses them. #[tokio::test] - async fn the_sealed_root_among_read_rows_is_refused() { + async fn the_rows_opened_under_another_column_are_refused() { let (cipher, _) = cipher().await; let doc = seal(&cipher, &fixture()).await; - let list = key(&cipher, "$.nested.list").await; - let mut rows = walk(&doc, &list); - rows.push(row(&doc, 0, Some(1))); - assertions::wrong_entry(open_value(&cipher, list.open(rows)).await, REACHED); + let nested = key(&cipher, "$.nested").await; + let opened = cipher + .decrypt_as::( + nested.open(walk(&doc, &nested)), + AeadContext::from(nonempty!("users").with("other")), + ) + .await; + assertions::aead(opened); } #[tokio::test] @@ -544,24 +772,19 @@ mod given_the_rows_of_a_read { let (cipher, _) = cipher().await; let doc = seal(&cipher, &json!({"arr": [10, 20, 30]})).await; let arr = key(&cipher, "$.arr").await; - // The read returns the array's own node, whose child count bounds - // its elements: leaving out the first would have opened as - // `[20, 30]` when the walk began at the children (review of - // #1358), and leaving out the last as `[10, 20]`. - let rows = walk(&doc, &arr); - let (first, last) = (rows[1].ordinal(), rows[rows.len() - 1].ordinal()); - for dropped in [first, last] { - let rows: Vec = walk(&doc, &arr) - .into_iter() - .filter(|n| n.ordinal() != dropped) - .collect(); - assertions::wrong_entry(open_value(&cipher, arr.open(rows.clone())).await, NEXT); - assertions::wrong_entry(open_list(&cipher, arr.open(rows)).await, NEXT); + // Leaving out the first would have opened as `[20, 30]` when the + // walk began at the children (review of #1358), and leaving out the + // last as `[10, 20]`: the array's ciphertext commits to all three. + for path in ["$.arr[0]", "$.arr[2]"] { + let dropped = slot(&cipher, &doc, path).await; + let rows = without(walk(&doc, &arr), dropped); + assertions::aead(open_value(&cipher, arr.open(rows.clone())).await); + assertions::aead(open_list(&cipher, arr.open(rows)).await); } } #[tokio::test] - async fn rows_with_no_reported_depth_are_placed_by_the_document() { + async fn rows_with_no_reported_depth_are_placed_by_the_chain() { let (cipher, _) = cipher().await; let doc = seal(&cipher, &fixture()).await; let nested = key(&cipher, "$.nested").await; @@ -575,7 +798,8 @@ mod given_the_rows_of_a_read { } /// What the read check allows by design, and says it allows: no bound field -/// names an entry's row, or the element a node at a selector sits under. +/// names an entry's row, or the element a node at a selector sits under, +/// and nothing above the node at the top of a read is read. mod given_answers_the_check_cannot_tell_apart { use super::*; @@ -598,13 +822,11 @@ mod given_answers_the_check_cannot_tell_apart { let (cipher, _) = cipher().await; let doc = seal(&cipher, &fixture()).await; let second = key(&cipher, "$.nested.list[1].x").await; - // `$.nested.list[0].x` (6) with `$.nested.list[1].x`'s self tag: - // the same selector and the same label, which is all a row binds. - let forged = vec![{ - let mut wire = serde_json::to_value(row(&doc, 6, Some(0))).unwrap(); - wire["x"] = json!(x_of(&doc, &second)); - serde_json::from_value::(wire).unwrap() - }]; + // `$.nested.list[0].x` with `$.nested.list[1].x`'s self tag at the + // top of a read: the same selector and the same label, which is all + // the node at the top binds. + let first = slot(&cipher, &doc, "$.nested.list[0].x").await; + let forged = with_x(vec![row(&doc, first, Some(0))], 0, x_of(&doc, &second)); let opened = open_value(&cipher, second.open(forged)).await; assert_eq!( opened.unwrap(), @@ -612,6 +834,20 @@ mod given_answers_the_check_cannot_tell_apart { "the first element's x, by design" ); } + + #[tokio::test] + async fn rows_withheld_whole_read_as_no_node() { + let (cipher, _) = cipher().await; + let nested = key(&cipher, "$.nested").await; + let opened = cipher + .decrypt_as::, JsonSubtree>(nested.open(Vec::new()), aead()) + .await; + assert_eq!( + opened.unwrap(), + None, + "the database's word that there is none" + ); + } } /// The four answers a read at a path with no wildcard gives, as `jsonb #>` @@ -668,11 +904,12 @@ async fn the_model_keeps_the_last_entry_carrying_a_tag_as_the_sql_index_does() { let doc = seal_with(&cipher, &fixture(), &open_root()).await; let root = key_with(&cipher, "$", &open_root()).await; let hello = key_with(&cipher, "$.hello", &open_root()).await; - // `hello` (1) appended again at the end: its self tag and parent tag - // now name two entries, 1 and 10. + let at = slot_with(&cipher, &doc, "$.hello", &open_root()).await; + // `hello` appended again at the end: its self tag and parent tag now + // name two entries, its own slot and 10. let mut wire = serde_json::to_value(&doc).unwrap(); let sv = wire["sv"].as_array_mut().unwrap(); - let copy = sv[1].clone(); + let copy = sv[at].clone(); sv.push(copy); let forged: JsonDocument = serde_json::from_value(wire).unwrap(); assert_eq!( @@ -687,7 +924,7 @@ async fn the_model_keeps_the_last_entry_carrying_a_tag_as_the_sql_index_does() { .collect(); assert_eq!(children.len(), 3, "the root's three members"); assert!( - children.contains(&10) && !children.contains(&1), + children.contains(&10) && !children.contains(&at), "the last entry with the parent tag: {children:?}" ); } @@ -706,3 +943,88 @@ fn with_x(mut rows: Vec, at: usize, x: String) -> Vec { rows[at] = serde_json::from_value(wire).unwrap(); rows } + +/// `rows` without the row of the entry at slot `j`. +fn without(rows: Vec, j: usize) -> Vec { + let before = rows.len(); + let rows: Vec = rows + .into_iter() + .filter(|n| n.ordinal() != j as u64) + .collect(); + assert_eq!(rows.len() + 1, before, "the row at {j} was among them"); + rows +} + +/// `node` carrying the self tag and parent tag of the entry at slot `j`. +fn with_tags_of(doc: &JsonDocument, node: JsonNode, j: usize) -> JsonNode { + let entry = &doc.entries()[j]; + let mut wire = serde_json::to_value(&node).unwrap(); + wire["x"] = json!(base64(entry.self_tag())); + wire["p"] = json!(entry.parent_tag().map(|p| base64(p))); + serde_json::from_value(wire).unwrap() +} + +/// `rows` with the self tags and parent tags of the rows of the entries at +/// slots `a` and `b` exchanged. +fn exchange_tags(rows: Vec, a: usize, b: usize) -> Vec { + let tags = |n: &JsonNode| { + let wire = serde_json::to_value(n).unwrap(); + (wire["x"].clone(), wire["p"].clone()) + }; + let find = |j: usize| rows.iter().position(|n| n.ordinal() == j as u64).unwrap(); + let (i, k) = (find(a), find(b)); + let ((xa, pa), (xb, pb)) = (tags(&rows[i]), tags(&rows[k])); + let retag = |n: &JsonNode, x: &Value, p: &Value| { + let mut wire = serde_json::to_value(n).unwrap(); + wire["x"] = x.clone(); + wire["p"] = p.clone(); + serde_json::from_value::(wire).unwrap() + }; + let mut rows = rows.clone(); + rows[i] = retag(&rows[i], &xb, &pb); + rows[k] = retag(&rows[k], &xa, &pa); + rows +} + +/// The sixteen bytes of a selector. +fn selector_bytes(selector: &JsonSelector) -> [u8; 16] { + use base64ct::{Base64, Encoding}; + Base64::decode_vec(&selector.to_string()) + .unwrap() + .try_into() + .unwrap() +} + +/// The chain's public hash functions, as the design writes them and the +/// server computes them: what a database holding a read's tokens can +/// compute for itself, and so what a forged row may carry. +mod chain { + use sha2::{Digest, Sha256}; + + fn h(parts: &[&[u8]]) -> [u8; 32] { + let mut hasher = Sha256::new(); + for part in parts { + hasher.update(part); + } + hasher.finalize().into() + } + + /// `H16("t" ‖ t(array) ‖ s(element) ‖ LE32(i))`. + pub(crate) fn element_token(array: &[u8; 16], s: &[u8; 16], i: u32) -> [u8; 16] { + h(&[b"t", array, s, &i.to_le_bytes()])[..16] + .try_into() + .unwrap() + } + + /// `H("x" ‖ iv ‖ t)[..8]`. + pub(crate) fn self_tag(iv: &[u8], token: &[u8; 16]) -> [u8; 8] { + h(&[b"x", iv, token])[..8].try_into().unwrap() + } + + /// `H("p" ‖ iv ‖ t(parent) ‖ LE32(o))[..8]`. + pub(crate) fn parent_tag(iv: &[u8], parent: &[u8; 16], o: u32) -> [u8; 8] { + h(&[b"p", iv, parent, &o.to_le_bytes()])[..8] + .try_into() + .unwrap() + } +} diff --git a/packages/stack-encrypt/tests/json_tree_vectors.rs b/packages/stack-encrypt/tests/json_tree_vectors.rs index 47465614a..0eb5cdc1a 100644 --- a/packages/stack-encrypt/tests/json_tree_vectors.rs +++ b/packages/stack-encrypt/tests/json_tree_vectors.rs @@ -1,10 +1,25 @@ -//! The JSON index's derivations, pinned: the selector `s`, the self tag `x` -//! and the parent tag `p` of every entry of one document, and the tokens a -//! read sends for the root, a chain head, an object member and an array -//! element, under both root options. Sealed under the deterministic key -//! source, whose index key is fixed and whose `iv` follows from its seed, -//! so every one of them is the same in every run. The ciphertexts carry a -//! fresh nonce each and are not pinned. +//! The JSON index's derivations, pinned, under both root options: +//! +//! - `entries`: the selector `s`, the self tag `x` and the parent tag `p` of +//! every entry, in the stored order; +//! - `order`: the path of the node stored at each slot, which is the +//! permutation the seeded generator drew (the root at slot 0); +//! - `selectors`: the multiset of stored selectors, sorted, which is what any +//! stored order shows; +//! - `q`: the containment projection as sets (every list sorted), which is +//! how `jsonb @>` reads it; +//! - `tokens`: the tokens a read sends for the root, a chain head, an object +//! member and an array element. +//! +//! Sealed under the deterministic key source, whose index key is fixed and +//! whose `iv` follows from its seed, with the stored order drawn from a +//! generator started at the same seed (`StackCipherBuilder::json_order_seed`, +//! behind the `test-support` feature, which this file needs), so every one +//! of them is the same in every run. The ciphertexts carry a fresh nonce +//! each and are not pinned. In production the order is drawn from the +//! operating system, so `entries` and `order` pin the generator's use here, +//! not a property of stored data; `selectors`, `q`, the tags and the tokens +//! are the same under any order. //! //! A stored document's `x` and `p` are derived from the tokens, and a read //! finds nodes by them, so a change to any derivation here is a change to @@ -13,6 +28,8 @@ //! regenerates nothing), deliberately not the crate's routine //! `STACK_ENCRYPT_UPDATE_FIXTURES`, and only when the break is intended. +#![cfg(feature = "test-support")] + mod common; use std::ffi::OsStr; @@ -44,6 +61,29 @@ fn document() -> Value { /// head, an array, an element, and a member below an element. const READS: [&str; 6] = ["$", "$.a", "$.a.b", "$.a.b[0]", "$.a.b[1]", "$.a.b[1].c"]; +/// Every node of [`document`], depth first. +const NODES: [&str; 7] = [ + "$", + "$.a", + "$.a.b", + "$.a.b[0]", + "$.a.b[1]", + "$.a.b[1].c", + "$.d", +]; + +/// A projection with every list sorted: `q` as the sets `jsonb @>` reads. +fn as_set(q: &Value) -> Value { + let mut sv = q["sv"].as_array().cloned().unwrap_or_default(); + sv.sort_by_key(|pair| pair.to_string()); + let mut el: Vec = q["el"] + .as_array() + .map(|groups| groups.iter().map(as_set).collect()) + .unwrap_or_default(); + el.sort_by_key(|group| group.to_string()); + json!({"sv": sv, "el": el}) +} + fn base64(bytes: &[u8]) -> String { use base64ct::{Base64, Encoding}; Base64::encode_string(bytes) @@ -73,6 +113,24 @@ async fn vectors(options: &JsonOptions) -> Value { }) }) .collect(); + let mut order = vec![Value::Null; doc.entries().len()]; + for path in NODES { + let key = cipher + .default_keyset() + .json_subtree_key( + &JsonPath::parse(path).unwrap(), + options, + nonempty!("users").with("profile"), + ) + .await + .unwrap(); + let (j, _) = doc + .find(&key.token_bytes()[0]) + .expect("every node is stored"); + order[j] = json!(path); + } + let mut selectors: Vec = doc.entries().iter().map(|e| base64(e.selector())).collect(); + selectors.sort_unstable(); let mut tokens = serde_json::Map::new(); for path in READS { let key = cipher @@ -89,6 +147,9 @@ async fn vectors(options: &JsonOptions) -> Value { json!({ "iv": base64(doc.header().iv()), "entries": entries, + "order": order, + "selectors": selectors, + "q": as_set(&serde_json::to_value(doc.projection()).unwrap()), "tokens": tokens, }) } @@ -105,7 +166,7 @@ say so in the changeset."; #[tokio::test] async fn the_derivations_are_pinned_under_the_deterministic_source() { let actual = json!({ - "_comment": "The JSON index's s, x and p per entry, and the tokens a read sends, under the deterministic key source. Read by tests/json_tree_vectors.rs; regenerate with STACK_ENCRYPT_UPDATE_JSON_TREE_VECTORS=1 only for an intended break. Do not edit by hand.", + "_comment": "The JSON index's s, x and p per entry in the stored order, the path stored at each slot (the permutation the seeded generator drew), the selector multiset, q as sets, and the tokens a read sends, under the deterministic key source. Read by tests/json_tree_vectors.rs; regenerate with STACK_ENCRYPT_UPDATE_JSON_TREE_VECTORS=1 only for an intended break. Do not edit by hand.", "document": document(), "sealed_root": vectors(&JsonOptions::default()).await, "open_root": vectors(&JsonOptions { sealed_root: false, ..JsonOptions::new() }).await, @@ -128,6 +189,12 @@ async fn the_derivations_are_pinned_under_the_deterministic_source() { pinned["sealed_root"]["iv"], actual["sealed_root"]["iv"], "the deterministic source's iv moved: a test-infrastructure change in stack-kms, not a stored-data break" ); + for root in ["sealed_root", "open_root"] { + assert_eq!( + pinned[root]["order"], actual[root]["order"], + "{root}: the stored order the seeded generator draws moved: a change to how the order is drawn (json::order) or to the generator, not a stored-data break" + ); + } assert_eq!(pinned, actual, "{DRIFT}"); } From 4440890779cba45230ba52a0e8abb0601f11a158 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 13:06:23 +1100 Subject: [PATCH 06/22] docs(json-tree): record the cascade and the random stored order The design record gains a dated note on the change (#1357): the cascade as the integrity rule and which refusal comes from where, the random stored order as the format and why it is derived from nothing, the wildcard first-match semantics the owner chose (an unspecified element, with the element-0 fix-up and the sealed root's heads digest considered and dropped), the keyset load on a root-open decrypt, and the two SQL doc comments that still say "document order". The leakage page drops its stored-order row, says what a random order leaves visible (the selector multiset, q's element grouping, the head count, ciphertext lengths and the ordering terms) and what it no longer shows, rewrites "What is authenticated" around the subtree guarantee with the three allowances kept, says what `->` returns at a wildcard path, and notes that a containment needle is still sent in key order. ADR 0010 says what a json_tree document authenticates and keeps the three allowances: another row's entry, another element's node at the top of a read, and withheld rows. The json module's rustdoc and eql-bindings' JsonTreeDocument doc stop describing a depth-first order. A changeset records the change for @cipherstash/eql. Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- .../eql-json-tree-cascade-and-random-order.md | 5 ++ docs/plans/2026-10-17-json-tree-index.md | 20 +++++++ .../crates/eql-bindings/src/v4/json_tree.rs | 3 +- .../eql/docs/reference/json-tree-leakage.md | 55 ++++++++++++------- .../0010-what-a-stored-value-authenticates.md | 32 ++++++++++- packages/stack-encrypt/src/json/mod.rs | 29 ++++++---- 6 files changed, 111 insertions(+), 33 deletions(-) create mode 100644 .changeset/eql-json-tree-cascade-and-random-order.md diff --git a/.changeset/eql-json-tree-cascade-and-random-order.md b/.changeset/eql-json-tree-cascade-and-random-order.md new file mode 100644 index 000000000..f38529834 --- /dev/null +++ b/.changeset/eql-json-tree-cascade-and-random-order.md @@ -0,0 +1,5 @@ +--- +"@cipherstash/eql": patch +--- + +EQL v4 `json_tree` (unreleased): a stored document's entries are now in a random order, root first, and its containment projection `q` is shuffled the same way, so the stored order no longer shows each row's tree, the size of every subtree or the byte order of sibling key names (#1357). Every entry's ciphertext now also commits to its children's AEAD tags, so the client checks a subtree read's rows by the chain rather than by their positions, and a row swapped, dropped, added, reordered or moved beneath the node read is refused by the cipher. Nothing stored changes shape and no SQL changes; documents sealed by the earlier build of this unreleased family no longer open. `->`, `->>` and an ordering on `eql_v4.ord_term(col -> sel)` at a wildcard path now return an unspecified element of the row (the same one every time), not element 0; read every element with `json_tree_path_query`. `docs/reference/json-tree-leakage.md` says what the stored row still shows and what a read now proves. diff --git a/docs/plans/2026-10-17-json-tree-index.md b/docs/plans/2026-10-17-json-tree-index.md index f8d869b20..8f043cca1 100644 --- a/docs/plans/2026-10-17-json-tree-index.md +++ b/docs/plans/2026-10-17-json-tree-index.md @@ -699,3 +699,23 @@ The gated-walk column is the benchmark's in-server figure for the same scenario, - *The sealed root's children enumerate from nothing.* The top-level entries carry no parent tag, because the format predates the root having a token, so `json_tree_children` at `$` returns no rows and `json_tree_child_count` returns NULL on the default layout, and a developer counts top-level fields by reading `$` and opening it. Chaining the heads from the root's token, `p = H("p" ‖ iv ‖ t($) ‖ LE32(o))`, would make the root enumerate like every other node. **Decided against for now (owner, 2026-10-25):** a walk from the root is a capability over the whole document's shape that the format deliberately withholds, so adding it increases leakage; SteVec offered nothing like it either, and it waits for a customer to ask. - *Withheld rows are not detected.* Nothing in a row says how many nodes a read should find, so a database that drops rows makes a node look absent, or shortens a wildcard read. This is an integrity property against a lying database, outside the current threat model (confidentiality against an honest-but-curious one); closing it needs an authenticated count, for instance the array's child count read beside a wildcard. - *The entries × tokens `LIMIT` and the token cap bound a lying database's own work.* Forged member tags can make a walk visit one subtree repeatedly; the client refuses the repeated positions, so only the database's work is at stake, and with the default sealed root and budget the read stops at `budget + 1` rows regardless. The `LIMIT` is the only bound when the budget is 0 or the root is open, and the token cap (`TermError::JsonWildcardTooWide`) scales it: a one-level wildcard with bound N sends N + 1 tokens and two levels send N² + N + 1, so the cap is 8,192 (owner, 2026-10-25; it was 65,536, which no measurement approached, the largest read benchmarked sending 4,161): every one-level read up to 8,191 elements, two levels up to a bound of 90, the default of 64 included, and no third level. + +### Implementation notes (2026-10-26): the cascade, and a random stored order (#1357) + +The stored order was the last disclosure of the format that no option changed: entries depth-first, members in key byte order, so a row's sequence of selectors gave its tree exactly, and comparing rows gave the parent of every node, every subtree's size and the byte order of sibling key names. It was deferred because the client's integrity check leaned on it: a read's rows had to sit at consecutive positions below the node asked for, with the position bound into each AAD. The spike in #1371 showed the way out, and the report linked from #1357 the analysis; this is the change, made the format. + +1. **The cascade is the integrity rule.** Every entry's AAD (domain `stack-encrypt/json/entry/v2`) binds, beside the keyset, the column, the key tag, the selector and the slot `j`, the digest of its children's AEAD tags, `SHA-256("k" ‖ tag(child₀) ‖ tag(child₁) ‖ …)` in ordinal order, a tag being the last sixteen bytes of a ciphertext as vitaminc lays a leaf out. Sealing runs from the leaves up. A leaf, an empty container and the sealed root (whose members chain from nothing) open under the digest of no children. A node's tag covers its AAD, so it commits to every ciphertext beneath it. Opening a read places its rows the way the server found them, by the chain (the node a token names by its self tag, a node's children by its parent tags at ordinals 0, 1, 2, …, each under the member or element rule its self tag fits), opens each node under the digest of the children placed under it, and then checks it. Nothing reads a slot as a position, so the rows may arrive, and be stored, in any order. Nothing stored changes: the AAD is not stored and the tag is already in `c`. Code: `packages/stack-encrypt/src/json/cascade.rs` and `place` in `derive.rs`. + + What it proves: the rows beneath the node a read names are exactly the ones sealed there, complete, each under its parent and in its place. What it does not: the place of that node itself, which is checked by its selector and its decrypted label alone, since nothing above it is read. The three allowances of ADR 0010 stand (another row's entry; another element's node at the same selector at the top of a read; withheld rows), and ADR 0010 now says so for `json_tree`. Upward verification was weighed in the report and not taken: it needs the ancestors' tokens, which is the capability the token-list read removed. + + Which refusal comes from where. The cipher (`Error::Aead`): a row swapped, dropped, added, reordered or moved under another parent beneath the node read; a container extracted alone with `->` (its AAD covers tags `->` does not return; it was `OpenedType` before); a truncated or reordered whole document; another column. The client (`Error::JsonWrongEntry`): rows of two documents and an entry returned twice, before any key is retrieved; a row no token's subtree reaches; a reported depth the chain does not give; a child whose self tag fits neither rule, or not the one its label names; a node at the top of a read whose selector or label is not its path's; the sealed root anywhere but at `$`; a container whose plaintext child count is not the number of children it opened over. The hostile-row tests (`tests/json_read_checks.rs`, the PostgreSQL oracle) assert each of these exactly, and find rows by their paths rather than their slots. + +2. **A random stored order is the format.** Not an option, a flag or a switch. The root stays at slot 0, where the sealed-root fallback (`sv -> 0`) finds it; every other entry is placed by a uniformly random permutation of the slots 1…n, and `q`'s pair lists and element groups are shuffled the same way. The permutation is drawn per document at sealing time and thrown away: it is derived from nothing, keyed or not, because nobody needs to recompute it, and a permutation derived from anything stored (the `iv`) would be computable by the database. The generator is vitaminc's `SafeRand::from_entropy` with `next_below` (Lemire's reduction, bias at most n / 2⁶⁴) driving Fisher–Yates (`packages/stack-encrypt/src/json/order.rs`); it is not part of the format, since nothing stored depends on it. For fixtures, `StackCipherBuilder::json_order_seed` (behind `test-support`) draws every document's order from one stream started at a seed, through `SafeRand::from_controlled_seed`, and the deterministic test cipher sets it: `tests/fixtures/json_tree_vectors.json` now pins, per root option, the entries in their stored order, the path stored at each slot, the selector multiset, `q` as sets, and the tokens. The tags and tokens did not move. + + What stays visible at rest, and no order hides: the multiset of selectors (paths, node counts, array lengths, whether two rows have the same paths in the same numbers), `q`'s grouping by array element, which entries are heads under the sealed root (the head count, no longer the size of each head's subtree), each ciphertext's length, and the ordering terms at ordered paths. The leakage page drops its stored-order row and says this; the executable profile in `tests/json_leakage` models the new order, reading every world in an order an observer can rebuild without the stored one (`canonical.rs`). One disclosure of the key order remains, at query time: a containment or equality needle is still sent in the walk's order, members in key byte order, so a needle with two sibling keys shows which sorts first. The page says so; shuffling needles on the client (the server reads them as sets) would close it, and is left for a decision of its own. + +3. **First match at a wildcard path is unspecified (owner, 2026-10-26).** `json_tree_path_query_first`, and so `->`, `->>` and an ordering on `eql_v4.ord_term(col -> sel)`, returns the first entry in `sv` carrying a selector. At a wildcard path (every element under `$.a[*]`, at any depth) that is now an unspecified element, stable for a stored row but not element 0. Ordering or comparing over the elements of a wildcard path is written over all the matches (`json_tree_path_query`, `EXISTS` or `ORDER BY` over their ordering terms), and element order within a document is recovered after decryption from the labels, as it already was. No SQL changes. Two variants were considered and dropped by the owner: a fix-up after the shuffle that kept each wildcard selector's document-first entry first (it would have revealed, per wildcard selector, which entry is the document-first one), and writing the heads' digest into the sealed root's plaintext so that a client reading the root with its heads could verify them. The sealed root commits to nothing beneath it. + +4. **The keyset is loaded on a root-open decrypt.** Opening a whole document whose root is open walks the chain from the root's token, which needs the index key of the keyset it was sealed under. The open asks for it beside the data key, as a new kind of request a `Pending` carries (`Request::load_keyset`), answered from the cipher's keyset cache or loaded from ZeroKMS on a miss, as `StackCipher::keyset` loads one. Settling a pending still asks only for a `DataKeySource`: the cipher captures the backend's `load_index_key` at `init`, where `IndexKeySource` is known. The sealed root (the default) loads nothing. + +Left as it was, and worth a follow-up: two SQL doc comments in `src/v3/json_tree/functions.sql` say rows come back "in document order" (`json_tree_path_query`, and `json_tree_subtree` within a token). Both return rows by slot (`ORDER BY` on `j`), which is now the random stored order; no caller depends on it, and the client places rows by the chain. They are left for a change that may touch SQL. The benchmarks were not re-run: nothing stored and nothing in SQL changed. The client's added work is one SHA-256 per container over its children's tags, at sealing and at opening, and at opening the parent-tag probes the server already runs, which replace the position checks. diff --git a/packages/eql/crates/eql-bindings/src/v4/json_tree.rs b/packages/eql/crates/eql-bindings/src/v4/json_tree.rs index 9364c45bb..9ebbccaf2 100644 --- a/packages/eql/crates/eql-bindings/src/v4/json_tree.rs +++ b/packages/eql/crates/eql-bindings/src/v4/json_tree.rs @@ -77,7 +77,8 @@ pub struct JsonTreeKeyHeader(pub Base64Text); /// `public.eql_v4_json_tree`: a JSON tree document, `{v, i, h, sv: /// [entry], q: group}`. Strict. `h` is the document's key header, stored /// once (every entry is sealed under the document's one data key); `sv` -/// holds one entry per node in depth-first order, the root first; `q` is +/// holds one entry per node, the root first and the rest in a random order +/// drawn when the document was sealed; `q` is /// the document's containment projection, one `{s, vs}` pair per node /// nested by array element, which the client writes beside the entries /// and which is the only place the value selectors live: the server's diff --git a/packages/eql/docs/reference/json-tree-leakage.md b/packages/eql/docs/reference/json-tree-leakage.md index a5130d0e5..6b248e88c 100644 --- a/packages/eql/docs/reference/json-tree-leakage.md +++ b/packages/eql/docs/reference/json-tree-leakage.md @@ -1,6 +1,6 @@ # What an encrypted JSON tree reveals -`public.eql_v4_json_tree` (EQL v4) stores a JSON document so that PostgreSQL can answer containment, path extraction, ordering at chosen paths and subtree reads without the key. Every value, key name and container sits inside an AES-256-GCM ciphertext under the document's data key, and nothing on this page shows the text of one. What the database can see is **structure**, and a great deal of it: which paths a row has and how many times, the order the nodes are stored in, which (path, value) pairs repeat across rows, and lengths. Each item below is what one query capability or one storage choice costs. Read this before you put a column on it, and decide the two options at the end with it in mind. +`public.eql_v4_json_tree` (EQL v4) stores a JSON document so that PostgreSQL can answer containment, path extraction, ordering at chosen paths and subtree reads without the key. Every value, key name and container sits inside an AES-256-GCM ciphertext under the document's data key, and nothing on this page shows the text of one. What the database can see is **structure**, and a great deal of it: which paths a row has and how many times, which nodes share an array element, which (path, value) pairs repeat across rows, and lengths. The order the nodes are stored in is random per row and shows none of it. Each item below is what one query capability or one storage choice costs. Read this before you put a column on it, and decide the two options at the end with it in mind. ## On this page @@ -17,45 +17,58 @@ Someone who holds the column and no key, such as a database administrator, a bac | field | what it reveals | |---|---| | `s`, the path selector on every node | **Which paths the row has, and how many nodes sit at each.** A selector is a keyed hash of the path, the same in every row of the column, so two rows with a selector in common hold the same path. Key names are inside the hash and not readable. Array elements share their array's selector, so `$.tags[0]` and `$.tags[1]` look alike, but each element is its own entry: the number of entries carrying `$.tags[*]`'s selector is the length of `$.tags` (the total over every array at that path, when the path runs through another array). | -| the order of the entries in `sv` | **The row's shape.** Entries are stored in depth-first order, the root first, each node followed by everything beneath it, and an object's members in the byte order of their key names' UTF-8 encodings (the order of serde_json's default map; a build that turns on serde_json's `preserve_order` feature stores them in insertion order instead, which shows the application's field order). So a row's sequence of selectors fixes its shape exactly: two rows with the same sequence have the same tree with the same key names, and two rows that differ show where. Which selector's path sits under which is not written down, but it is the same in every row, so an observer comparing rows learns it, and with it each row's parent and child relations and the node count of every subtree. Siblings stored in key order also show, across rows, **the byte order of sibling key names** (for example that the key behind one selector sorts before the key behind another; byte order, not a locale's alphabetical order, so `Z` sorts before `a`), which narrows guesses at what each selector names. | -| `q`, the containment projection: one `{s, vs}` pair per node, nested by array element, in the same order | **Whether two rows hold the same value at the same path**, through the value selector `vs`, a keyed hash of the path and the value (for an object or an array, of its kind alone). So at each path the database sees which rows agree and how often each value occurs: a frequency histogram of every path, and whether one row's document contains another's. Not the value itself, and not whether two different paths hold the same value (the ordering terms below do show that, at the paths the index orders). **Which nodes sit inside the same array element**, through the nesting, for every element that is an object or an array. That grouping is what makes containment exact for arrays of objects. v3 reveals it too, through its entry order and per-element `a` flag (see the comparison below); here it is explicit in the projection. | -| `p`, the parent tag below the chain's heads | Nothing beyond the entry order on its own. It is salted by the document's `iv`, so the same position in two rows is unrelated bytes, and it does not say which entry is the parent. Which entries carry no `p` does show: with the sealed root on, those are the root's direct members, and since everything beneath a member follows it, **the node count of each top-level field's subtree** can be read off a single row exactly. With the sealed root off, only the root lacks one. | +| `q`, the containment projection: one `{s, vs}` pair per node, nested by array element, each list in a random order | **Whether two rows hold the same value at the same path**, through the value selector `vs`, a keyed hash of the path and the value (for an object or an array, of its kind alone). So at each path the database sees which rows agree and how often each value occurs: a frequency histogram of every path, and whether one row's document contains another's. Not the value itself, and not whether two different paths hold the same value (the ordering terms below do show that, at the paths the index orders). **Which nodes sit inside the same array element**, through the nesting, for every element that is an object or an array, and which elements sit inside which. That grouping is what makes containment exact for arrays of objects. v3 reveals it too, through its entry order and per-element `a` flag (see the comparison below); here it is explicit in the projection. The pairs of each group and the groups of each list are stored in a random order, so the projection does not restate the document's order. | +| `p`, the parent tag below the chain's heads | Nothing on its own. It is salted by the document's `iv`, so the same position in two rows is unrelated bytes, and it does not say which entry is the parent. Which entries carry no `p` does show: with the sealed root on, those are the root's direct members, so **how many top-level fields the row has**, and which selectors they carry. Not what sits beneath each one: the entries are stored in a random order, so nothing ties an entry to its top-level field. With the sealed root off, only the root lacks one. | | `x`, the self tag on every entry | Nothing on its own. It is a hash of the entry's own token salted by the document's `iv`, so it is unrelated bytes in every row and links no entry to another, and since every entry carries one, its presence marks none. Given a token, it is how the database finds that token's node in any row. | | each ciphertext's length, and the number of entries | **The node count**, and for each node the length of its key name (for an object member) plus, for a scalar, the length of its value, exactly, less the AEAD's fixed overhead. A string's length in bytes and a number's length in digits are therefore visible per node. With the sealed root on, the root entry's length is the document's length as JSON. | | `op`, the ordering term, only on number and string leaves at the paths the index orders | **The order of that leaf's value among every row that holds one, and which rows hold equal values.** The term is order-preserving encryption (CLLW OPE) under one key per column for each kind of leaf, so leaves at two ordered paths compare with each other too, and two ordered paths holding equal values carry equal terms. A number's equality is as a 64-bit float: two different integers past 2⁵³ can round to one float, and then they share a term and tie under `ORDER BY` while their value selectors differ. Strings are ordered and compared after their case and accents are folded, so `"Abc"`, `"abc"` and `"ábc"` get the same term: the equality it shows is of folded values. It is deterministic and not a random value: two values whose encodings share leading bits (for a number, its sign and then its exponent's leading bits; for a string, its first characters) get terms that share leading bytes, one byte per bit, so the term also shows how long that common prefix is, to the bit. So the stored terms alone sort the numbers at the ordered paths into signs and binary orders of magnitude, and show how far into their folded text two strings agree. This is CLLW's known profile; a scheme that hides the prefix is planned for a later version. **Its length is the leaf's length**: one byte per bit of the value's order encoding plus one, so 65 bytes for every number and `8 × (n + 1) + 1` bytes for a string whose folded UTF-8 form is `n` bytes. A term's presence at an ordered path also says the leaf is a number or a string. At an ordered path whose values are numbers in some rows and strings in others, the order between a number and a string depends on the column key and is not jsonb's (jsonb sorts numbers before strings); within each kind the order is the values'. The two kinds' terms are under different keys, so the terms also show which leaves are numbers and which are strings, and the strings sort together, as one block that the key places below the numbers, between the negative and the non-negative ones, or above them. By default no path is ordered; see the options below. | | `h`, the header | The keyset the data key was minted under, the data key's identifier (unique to the row), and whether the root is sealed. `v` and `i` are the version and the table and column identifier every EQL value carries. | -Not revealed at rest: any value's or key name's text, and, except through the ordering terms at the paths the index orders, whether two different paths hold the same value. Revealed at rest, by the entry order, although no field stores it: the shape of every row, the parent of every node, every array's length, and the byte order of sibling key names. +The entries of `sv` are stored with the root first, where the database finds the sealed root, and every other entry in a uniformly random order drawn when the row is sealed and then discarded. It is not derived from anything stored or from any key, so it is unrelated in every row, two rows sealed from one document included, and it shows nothing beyond the root's place. + +Not revealed at rest: any value's or key name's text; except through the ordering terms at the paths the index orders, whether two different paths hold the same value; and, since the entries are stored in a random order, the row's tree as such: no row writes down the parent of a node, the size of a subtree, or which of two sibling key names sorts first. + +What stays visible whatever the order: + +- **the multiset of selectors**: which paths the row has and how many nodes sit at each, so every array's length, and whether two rows have the same paths in the same numbers; +- **`q`'s grouping by array element**: which nodes share an element that is an object or an array, and which elements sit inside which; +- **the head count** under the sealed root: how many top-level fields the row has; +- **each ciphertext's length**; +- **the ordering terms**, at the paths the index orders. + +From these the tree can still be partly inferred across rows: a path never occurs without its parent's, so which selectors occur together suggests which path sits under which, and inside array elements the grouping in `q` says it outright. ## What a query shows A query hands the database a term derived from the plaintext it asks about, and the database's answer is visible to it. Each query type reveals the following, in addition to the rows it returns. -- **Containment (`@>`, `<@`), and equality at a path, which is sent as a containment needle.** The needle's own projection: the selectors of its paths, the value selectors of its nodes, and its element grouping. So the database learns which paths and which (path, value) pairs you asked about, and which rows hold them. -- **Path extraction (`->`, `->>`, `json_tree_path_query`, `json_tree_path_exists`).** The selector of the path asked. A position in the path reads as the wildcard, so a position is not sent. The entry returned is a ciphertext. +- **Containment (`@>`, `<@`), and equality at a path, which is sent as a containment needle.** The needle's own projection: the selectors of its paths, the value selectors of its nodes, and its element grouping. So the database learns which paths and which (path, value) pairs you asked about, and which rows hold them. Unlike the stored projection, a needle is sent in the order the client walks it, an object's members in the byte order of their key names, so a needle with two sibling keys shows which of them sorts first, at selectors the database also sees in the stored rows. +- **Path extraction (`->`, `->>`, `json_tree_path_query`, `json_tree_path_exists`).** The selector of the path asked. A position in the path reads as the wildcard, so a position is not sent. The entry returned is a ciphertext. At a wildcard path (any path with an array step, at any depth: every element under `$.orders[*]`), `->`, `->>` and an ordering on `eql_v4.ord_term(col -> sel)` take the first entry carrying the selector in the stored order, which is random per row: an unspecified element, the same one every time that row is read, and not element 0. To compare, order or read every element, use the all-match form (`json_tree_path_query`, or `EXISTS` and `ORDER BY` over the matches' ordering terms); a client recovers element order after decryption, from each element's position, which is inside its ciphertext. - **Ordering at a path (`<`, `<=`, `>`, `>=`).** The path's selector and the bound's own ordering term, which compares with the stored terms exactly as they compare with each other: the database learns the bound's place in the order of the stored values at every ordered path, and whether it equals one. `ORDER BY ord_term(…)` sends no term and shows nothing the stored terms do not. - **A subtree read (`json_tree_subtree`, `json_tree_children`, `json_tree_child_count`).** A **token** for exactly the node the path names, which the client derives from the path alone. Reading `$.profile.address.city` sends that leaf's token and nothing that covers `$.profile`. No token over a top-level field or an array leaves the client unless that field or array is itself what is read. - - **A wildcard** (`$.items[*].price`) is expanded by the client into one token per concrete path, up to a bound `N` (64 by default, set by the caller), plus a **probe** token for the element at index `N` of every array a wildcard expands. A probe that finds an element means the array is longer than the bound, and the read fails rather than return part of the answer. So the database learns which of the nodes exist in each row, which shows, up to `N`, which elements hold the rest of the path (each array's length, where the path ends at the element; the entry order already shows it), whether each array is longer than `N`, and the structure beneath each node it finds. It learns nothing about the elements' sibling fields. In a row where a probe finds an element, the database also walks that element's subtree, although the read then fails. That exception is in the token, not in what the SQL function does with it: a probe is the element's own token, so the database can find that element in any row, enumerate beneath it and derive its descendants' tokens, in rows written later too, whatever the function returns. It holds for any read that sends the element's token. Two wildcard levels send `N² + N + 1` tokens, `N + 1` of them probes, one at index `N` of each array the read expands: the `N` inner arrays and the outer one. + - **A wildcard** (`$.items[*].price`) is expanded by the client into one token per concrete path, up to a bound `N` (64 by default, set by the caller), plus a **probe** token for the element at index `N` of every array a wildcard expands. A probe that finds an element means the array is longer than the bound, and the read fails rather than return part of the answer. So the database learns which of the nodes exist in each row, which shows, up to `N`, which elements hold the rest of the path (each array's length, where the path ends at the element; the selectors already show it), whether each array is longer than `N`, and the structure beneath each node it finds. It learns nothing about the elements' sibling fields. In a row where a probe finds an element, the database also walks that element's subtree, although the read then fails. That exception is in the token, not in what the SQL function does with it: a probe is the element's own token, so the database can find that element in any row, enumerate beneath it and derive its descendants' tokens, in rows written later too, whatever the function returns. It holds for any read that sends the element's token. Two wildcard levels send `N² + N + 1` tokens, `N + 1` of them probes, one at index `N` of each array the read expands: the `N` inner arrays and the outer one. - **What a token is.** A capability over the structure of its node's subtree in *every row of the column*: whoever holds it can find that node in any row (through its self tag `x`), see how the nodes beneath it nest, link each one to its parent, and derive the tokens of the nodes beneath. It also learns whether each node beneath is an object member or an array element, and each element's index: nothing stored says which a node is, but to derive a node's token the database tests its self tag against the member rule and then the element rule, and the one that matches says which; an element's index is the ordinal it is found at. It opens no value. Tokens are column-wide and do not expire, so a token sent once is useful to the database for as long as the column lives. Send a token to the database in a query and nowhere else. - **The root.** A read of `$` sends the root's token, and a read of anything below `$` does not. With the sealed root on (the default) that token covers no document's structure: it finds the root entry, whose ciphertext is the document, and nothing beneath it, since the top-level fields chain from nothing. With it off, the root's token covers every row's whole document. - **`json_tree_children` and `json_tree_child_count`** take one token, of any node a path names, and send nothing else. Under the sealed root the root's token enumerates nothing (its count is NULL), for the same reason. ## What is authenticated -Only the ciphertext `c`. Opening an entry verifies that it was sealed under this keyset, for this column, under the data key its row's header names (a fresh one for every row), with the selector `s` and the position `j` it is opened with; a ciphertext moved into another row or column, or opened under another selector or position, fails. Nothing in it names a row, so an entry that travels with its own row's header still opens; see below. When the client opens a whole document it takes each entry's position from its place in `sv`, so a moved, swapped or re-labelled entry fails to open. The selectors, the projection, the parent tags, the self tags, the ordering terms, the header's sealed-root flag, and which entries a row holds are not authenticated, by the same rule every EQL stored value follows (ADR 0010 in `packages/stack-encrypt/docs/adr`). +Only the ciphertext `c`. Opening an entry verifies that it was sealed under this keyset, for this column, under the data key its row's header names (a fresh one for every row), with the selector `s` and the slot `j` it is opened with, and over the AEAD tags of exactly the children it is opened with: each entry's AAD binds a digest of its children's tags, in order. A ciphertext moved into another row or column, or opened under another selector, slot or set of children, fails. Nothing in it names a row, so an entry that travels with its own row's header still opens; see below. The selectors, the projection, the parent tags, the self tags, the ordering terms, the header's sealed-root flag, and which entries a row holds are not authenticated, by the same rule every EQL stored value follows (ADR 0010 in `packages/stack-encrypt/docs/adr`). + +**What a subtree read proves.** A node's tag covers its AAD, so it covers its children's tags, which cover theirs: the tag of a node commits to every ciphertext beneath it. So for a read that returns a node and everything beneath it, the rows beneath that node are exactly the ones sealed there, complete, each under the parent it was sealed under and in its place among its siblings, whatever order the rows arrive in. The client finds where each row should sit from the tags the server uses (the node a token names by its self tag `x`, a node's children by its parent tags `p`) and opens each node over the tags of the children found there; a row swapped, dropped, added, repeated under another parent, reordered or moved breaks a tag on the way up, and the node above it is refused by the cipher (`Aead`). A whole document opened with its root open is checked the same way, from the root down. The sealed root commits to nothing beneath it: its ciphertext is the whole document. -The consequence to design around: tampering with those fields changes **which rows a query returns, where a walk stops, and which entries of a row an extraction or a walk hands back**. An entry the database returns from `->` or a walk carries its own `s` and `j`, and the header it carries names the data key, so a decrypted value is always one that was sealed in this column. The client also checks that it is the entry it asked for, from what it derived for the query, and refuses anything else (`JsonWrongEntry`): +The consequence to design around: tampering with the unauthenticated fields changes **which rows a query returns, where a walk stops, and which entries of a row an extraction or a walk hands back**. An entry the database returns from `->` or a walk carries its own `s` and `j`, and the header it carries names the data key, so a decrypted value is always one that was sealed in this column. The client also checks that what it got is what it asked for, from what it derived for the query, and refuses anything else (`JsonWrongEntry`): -- **`->`.** The entry must carry the selector the client derived for the path. Another path's entry, such as the row's `$.bonus` returned for `$.salary`, is refused, and so is the sealed root returned for a field (it opens to the whole document); the root is accepted only when `$` itself was asked for. -- **A subtree read.** The node each token names, returned at depth 0, must carry that path's selector and, in its decrypted label, the key or position the path ends in (the root, for `$`). Every row beneath it must sit at the next position `j` after the row before it, with no gap, and carry the selector of its parent's path extended by its own decrypted key (or the wildcard, for an array element). A row of another path, another position of the same array, a dropped, truncated or repeated row, a row whose parent is not among the rows, and rows of two documents are refused. Each row's self tag `x` must also be the one the client recomputes from the token it sent, or, below depth 0, from its parent's token and the row's own selector and label. The self tags are not authenticated, and the database holds every token a read sends and every row's `iv`, so it can write any self tag it likes: the `x` check is a consistency check, which catches a corrupted row or a server that walked wrongly, not a substitution. The selector, the position `j` and the label, which the AAD and the ciphertext cover, are the security check. A read of a wildcard whose array is longer than the bound fails rather than return part of the list, both when the rows are walked and when they fall back to the sealed root. +- **`->`.** The entry must carry the selector the client derived for the path. Another path's entry, such as the row's `$.bonus` returned for `$.salary`, is refused, and so is the sealed root returned for a field (it opens to the whole document); the root is accepted only when `$` itself was asked for. A container with children does not open alone, since its AAD covers tags `->` does not return; the cipher refuses it, and its value is read with a subtree read. +- **A subtree read.** The node each token names, returned at depth 0, must carry that path's selector and, in its decrypted label, the key or position the path ends in (the root, for `$`): nothing above it is read, so these checks are all that place it. Beneath it the cascade places every row. The client also refuses, before the cipher is asked, rows of two documents and an entry returned twice; and after it, a row no token's subtree reaches, a depth the server reports that is not the one the chain gives the row, and a child whose self tag fits neither the member nor the element rule, or not the one its decrypted label names. The self tags and parent tags are not authenticated, and the database holds every token a read sends and every row's `iv`, so it can write any tag it likes: the tags say where to try each row, and the cipher decides. A read of a wildcard whose array is longer than the bound fails rather than return part of the list, both when the rows are walked and when they fall back to the sealed root. What the checks do not cover, by design: - **Which row an entry comes from.** Nothing in an entry names its row, so another row's `$.salary`, returned with that row's header, opens; that is the rule every EQL stored value follows (ADR 0010). -- **Which element a node sits under.** Every element of an array shares its selector, and no field a row binds says which element a node belongs to, so a read of `$.items[1].price` can be answered with `$.items[0].price` of the same row (its self tag rewritten), and opens to that element's price. -- **Which nodes a row holds.** Nothing in a row says how many nodes a read should find, so a database can withhold rows: a read answered with none opens as no node at the path (`None`), and a wildcard read answered with some elements' rows left out opens to the rest. That includes a probe's rows, so an array longer than the bound can open as its first `N` elements instead of failing. A node it does return is checked as above. +- **Which element the node at the top of a read sits under.** Every element of an array shares its selector, and nothing the node binds says which element it belongs to, so a read of `$.items[1].price` can be answered with `$.items[0].price` of the same row (its self tag rewritten), and opens to that element's price. Beneath the top of a read the cascade rules this out; at the top, ruling it out would mean checking the node against its ancestors, which the read does not return. +- **Which nodes a row holds.** Nothing in a row says how many nodes a read should find, so a database can withhold rows: a read answered with none opens as no node at the path (`None`), and a wildcard read answered with some elements' rows left out opens to the rest. That includes a probe's rows, so an array longer than the bound can open as its first `N` elements instead of failing. A node it does return comes with its whole subtree or is refused. -A query's result is not integrity-protected as a whole: each decrypted value is genuine and at a path with the selector asked for, but which row it came from, under which element, and whether anything was left out, is the database's word. +A query's result is not integrity-protected as a whole: each decrypted value is genuine, at a path with the selector asked for and with everything beneath it as sealed, but which row it came from, under which element, and whether anything was left out, is the database's word. ## Compared with EQL v3 (`public.eql_v3_json_search`) @@ -65,19 +78,19 @@ A query's result is not integrity-protected as a whole: each decrypted value is | which (path, value) pairs a row has | yes | yes | | node count | yes, two entries per node | yes, one entry per node | | array lengths | yes | yes, from how many entries carry the element selector | -| size of every subtree | yes: each path entry's ciphertext is the subtree under it | no ciphertext spans a subtree, but its node count follows from the entry order, and its size is the sum of its entries' lengths | -| parent and child relation | inferable from subtree sizes and co-occurrence across rows | not stored, but inferable from the entry order and co-occurrence across rows; exact for top-level fields under the sealed root | -| whether two rows have the same shape | yes, selectors are column-deterministic | yes, selectors are column-deterministic and entries are stored in depth-first order | -| which leaves sit in the same array element | yes, whenever elements are indexed at all. Entries sit in document order and every element's entry carries the `a` flag (`parent_is_array`), so the leaves between two flagged entries belong to the element before them; with position selectors (`array_index_mode` `position` or `all`) each leaf's selector names its element outright. Only `array_index_mode: none`, the stack's default, hides it, by indexing nothing inside an array | yes, through the projection's nesting and the entry order, as v3's order and flag do | +| size of every subtree | yes: each path entry's ciphertext is the subtree under it | no: no ciphertext spans a subtree and the entries are stored in a random order; the node count at each path is visible | +| parent and child relation | inferable from subtree sizes and co-occurrence across rows | not stored, and not in the order; inferable from which paths occur together across rows, and explicit inside array elements through the projection's nesting | +| whether two rows have the same shape | yes, selectors are column-deterministic | as far as their paths, their node counts at each path and their element grouping go: selectors are column-deterministic, and the stored order is random | +| which leaves sit in the same array element | yes, whenever elements are indexed at all. Entries sit in document order and every element's entry carries the `a` flag (`parent_is_array`), so the leaves between two flagged entries belong to the element before them; with position selectors (`array_index_mode` `position` or `all`) each leaf's selector names its element outright. Only `array_index_mode: none`, the stack's default, hides it, by indexing nothing inside an array | yes, through the projection's nesting, as v3's order and flag do | | equal values in sibling elements | ciphertexts equal | ciphertexts differ | | ordering terms | on scalar leaves, with no per-path choice | only at the paths the index orders; none by default | | a subtree read | reveals the selector asked | reveals a token for exactly the node asked for (one per concrete path, and one probe for each array a wildcard expands), and with it that node's shape in every row for the life of the column | ## The two options that change the picture -- **`sealed_root`** (default on). The root entry's ciphertext is the whole document, so a full read is one decryption and no token a query sends covers a whole document: the root's token finds the root entry and nothing beneath it. The costs are that the root entry's length reveals the document's size, the row is larger, and the root's direct members are marked by their missing `p`, which puts each top-level field's node count in plain view. Off, there is no document-sized ciphertext, but a read of `$` itself sends the root's token, the whole column's shape. +- **`sealed_root`** (default on). The root entry's ciphertext is the whole document, so a full read is one decryption and no token a query sends covers a whole document: the root's token finds the root entry and nothing beneath it. The costs are that the root entry's length reveals the document's size, the row is larger, and the root's direct members are marked by their missing `p`, which shows how many top-level fields the row has. Off, there is no document-sized ciphertext, but a read of `$` itself sends the root's token, the whole column's shape. - **`ordered`** (default none). Each path listed carries an ordering term and takes on the `op` row above; a path not listed cannot be range-queried or sorted, and a query that tries is refused by the client before any key loads. Order only the paths your queries sort or range over. -One disclosure above is a property of the stored format that no option changes today: the depth-first entry order, with members in sorted key order. It is an open decision recorded in the design document (`docs/plans/2026-10-17-json-tree-index.md` at the repository root, Phase 4 notes) and in #1357. +Neither option changes the stored order: it is random in every row, which is what took the row's tree out of the stored view (#1357; the design document, `docs/plans/2026-10-17-json-tree-index.md` at the repository root, records why). To report a concern with any of this, see `SECURITY.md` at the repository root. diff --git a/packages/stack-encrypt/docs/adr/0010-what-a-stored-value-authenticates.md b/packages/stack-encrypt/docs/adr/0010-what-a-stored-value-authenticates.md index b53aed707..7dc8ed691 100644 --- a/packages/stack-encrypt/docs/adr/0010-what-a-stored-value-authenticates.md +++ b/packages/stack-encrypt/docs/adr/0010-what-a-stored-value-authenticates.md @@ -1,6 +1,7 @@ --- status: accepted date: 2026-10-18 +amended: 2026-10-26 extends: vitaminc ADR 0002 --- @@ -40,7 +41,36 @@ was sealed exactly so, under exactly that context. rows. A caller who needs a value bound to its row puts the row's identity (or a tenant) in the context. -## Consequences +### The JSON index (`json_tree`) + +A JSON index document is one sealed entry per node, and it authenticates +more than a container does. Each entry's ciphertext binds, beside its +context, the node's path selector, the slot it is stored at, and a digest +of its children's AEAD tags in their order (amended 2026-10-26, the +cascade). A node's tag therefore commits to every ciphertext beneath it: +the client opens the rows beneath the node a read names only if they are +exactly the ones sealed there, each under its parent and in its place, +however the database orders them. A row swapped, dropped, added, +reordered or moved under another parent is `Error::Aead`. The stored +order of the entries is random and carries no meaning, and the search +fields beside the ciphertexts (selectors, the projection, the self and +parent tags, the ordering terms) are not authenticated, like every +search term. + +Three things stay outside it, by design, as they were before the cascade: + +- **Another row's entry.** An entry returned with its own row's header + opens, as any leaf sealed under the column context does (the row rule + above). +- **Another element's node at the top of a read.** Every element of an + array shares a selector, and the node a read names is checked by its + selector and label alone, since nothing above it is read. So + `$.items[0].price` opens in answer to a read of `$.items[1].price`. + Beneath that node the cascade rules the substitution out. +- **Withheld rows.** Nothing says how many nodes a read should find, so a + read the database answers with fewer whole subtrees, or none, opens to + what it returned. + - **A typed open separates a failed leaf from a refused shape.** A leaf that does not open (wrong key, wrong context, altered bytes) is diff --git a/packages/stack-encrypt/src/json/mod.rs b/packages/stack-encrypt/src/json/mod.rs index bf328b1b2..95e4b5fec 100644 --- a/packages/stack-encrypt/src/json/mod.rs +++ b/packages/stack-encrypt/src/json/mod.rs @@ -62,9 +62,12 @@ //! client derived for the read: [`JsonSelector::open`] for the one entry //! `->` returns ([`JsonExtracted`]), [`JsonSubtreeKey::open`] for the rows //! of a walk ([`JsonSubtree`]). Each checks that the rows are the ones the -//! read asked for, which their ciphertexts alone do not say: an entry is -//! sealed under its own selector and position, so any entry of the row -//! opens. +//! read asked for. Every entry's AAD binds its selector, the slot it is +//! stored at and the digest of its children's AEAD tags (the cascade), so +//! the rows beneath the node a read names open only as they were sealed; +//! the node itself is checked against the path the client derived, since +//! nothing above it is read. The entries are stored in a random order, root +//! first, and no check reads a slot as a position in the document. //! //! # Where this deviates from the design //! @@ -78,9 +81,10 @@ //! ciphertext), so there is one leaf cipher in the crate and no second //! AES-GCM call site. Uniqueness under one document key is probabilistic //! rather than structural: at `N` entries the collision probability is -//! `N² / 2^97`, negligible for any document. The ordinal still binds the -//! entry to its place through the AAD, so a moved or reordered entry fails -//! to open as the design requires. The cost is twelve bytes per entry. +//! `N² / 2^97`, negligible for any document. The AAD still binds the +//! entry to its slot, and its parent's AAD to its tag, so a moved or +//! reordered entry fails to open as the design requires. The cost is +//! twelve bytes per entry. //! - **Tokens chain by selector, and nothing stores a token.** The design //! wraps each child's label-derived token under its parent's (`w`) and //! stores the ordinal (`o`) so the server can find and unmask it. Here a @@ -515,7 +519,9 @@ impl JsonDocument { /// The rows the server's read returns for `tokens` with no budget: /// for each token in order, the node it names at depth `0` and every - /// entry beneath it with its depth below that node, in document order. + /// entry beneath it with its depth below that node, each node followed + /// by its children in ordinal order. (The SQL returns the rows of a + /// token by slot instead, which the client does not rely on.) /// A token that names no node yields nothing. What /// `json_tree_subtree(doc, tokens, 0)` returns. /// @@ -597,8 +603,9 @@ where } /// One entry as an extraction returns it: the entry, the document's header -/// grafted beside it, and the entry's ordinal `j` in the document, which -/// its ciphertext is bound to. What `->` and the walk return per row. +/// grafted beside it, and the slot `j` the entry is stored at in the +/// document, which its ciphertext is bound to. What `->` and the walk +/// return per row. /// /// A row is the database's word, so it does not open on its own: the /// selector the client derived opens the entry `->` returned @@ -626,7 +633,9 @@ impl JsonNode { pub fn header(&self) -> &JsonHeader { &self.header } - /// The entry's ordinal in the document. + /// The slot the entry is stored at in the document, `j` on the wire. + /// The stored order is random, so it is not the node's place in the + /// document. pub fn ordinal(&self) -> u64 { self.j } From 76902198de446d0581752c5d3376594d8a027e26 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 13:32:57 +1100 Subject: [PATCH 07/22] fix(stack-encrypt): tighten the keyset load and compare projections as sets From the review of #1379: - A keyset request is now answered once per keyset in a batch, however many documents ask: a column of root-open documents under one keyset costs one cache lookup, or one ZeroKMS load on a miss, not one each. - A load by id that ZeroKMS (or a custom source) answers for another id is refused as KeysetMismatch. It would otherwise walk the document under another keyset's PRF and cache the state under an id no later lookup asks for. - The three places that built a KeysetState from an index key share one constructor. - JsonProjection's equality is now as sets, the way `jsonb @>` reads a projection: two seals of one document carry projections that differ only in their shuffled order, and compared unequal under the derived equality. - Opening a whole root-open document moves its entries into the rows instead of cloning every ciphertext. - `no_children`'s rustdoc said the sealed root commits to its heads in its plaintext, which this change does not do. It commits to nothing beneath it; the heads are each checked as the top of a read. Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- packages/stack-encrypt/src/cipher.rs | 40 ++++++--- packages/stack-encrypt/src/json/cascade.rs | 5 +- packages/stack-encrypt/src/json/derive.rs | 21 +++-- packages/stack-encrypt/src/json/query.rs | 92 ++++++++++++++++++- packages/stack-encrypt/src/target/pending.rs | 94 +++++++++++++++++++- 5 files changed, 222 insertions(+), 30 deletions(-) diff --git a/packages/stack-encrypt/src/cipher.rs b/packages/stack-encrypt/src/cipher.rs index 5468af0b2..7419f7292 100644 --- a/packages/stack-encrypt/src/cipher.rs +++ b/packages/stack-encrypt/src/cipher.rs @@ -639,12 +639,17 @@ impl StackCipher { Lookup::Hit(state) => return Ok(state), Lookup::Stale(resolution) | Lookup::Miss(resolution) => resolution, }; - let (id, index_key) = (self.index_keys)(&self.kms, IdentifiedBy::Uuid(id)).await?; - let state = Arc::new(KeysetState { - id, - name: None, - prf: hmac_prf_from_index_key(&index_key), - }); + let (loaded, index_key) = (self.index_keys)(&self.kms, IdentifiedBy::Uuid(id)).await?; + // Asked by id, ZeroKMS answers for that id. A source that answers for + // another would have the document walked under another keyset's PRF, + // and cached under an id no later lookup asks for. + if loaded != id { + return Err(Error::KeysetMismatch { + left: id, + right: loaded, + }); + } + let state = keyset_state_of(loaded, None, &index_key); Ok(self.keysets().insert(state, resolution)) } @@ -762,11 +767,22 @@ async fn load_keyset( IdentifiedBy::Uuid(_) => None, }; let (id, index_key) = kms.load_index_key(Some(keyset)).await?; - Ok(Arc::new(KeysetState { + Ok(keyset_state_of(id, name, &index_key)) +} + +/// The state the cipher holds for a loaded keyset: its id, the name it was +/// selected by, and the PRF keyed by its index key. One construction for +/// every way a keyset is loaded. +fn keyset_state_of( + id: Uuid, + name: Option, + index_key: &stack_kms::IndexKey, +) -> Arc { + Arc::new(KeysetState { id, name, - prf: hmac_prf_from_index_key(&index_key), - })) + prf: hmac_prf_from_index_key(index_key), + }) } /// The client's own default keyset — the one a ZeroKMS administrator set for @@ -775,11 +791,7 @@ async fn load_keyset( /// there is no id or name to carry. async fn load_default_keyset(kms: &K) -> Result, Error> { let (id, index_key) = kms.load_index_key(None).await?; - Ok(Arc::new(KeysetState { - id, - name: None, - prf: hmac_prf_from_index_key(&index_key), - })) + Ok(keyset_state_of(id, None, &index_key)) } /// The state of a [`StackCipherBuilder`] that has not been given a data-key diff --git a/packages/stack-encrypt/src/json/cascade.rs b/packages/stack-encrypt/src/json/cascade.rs index b56337761..946b9c0b1 100644 --- a/packages/stack-encrypt/src/json/cascade.rs +++ b/packages/stack-encrypt/src/json/cascade.rs @@ -63,8 +63,9 @@ pub(crate) fn children_digest<'c>(children: impl IntoIterator) /// The digest of no children: what a leaf, an empty container and the sealed /// root open under. The sealed root's ciphertext is the whole document, and -/// the budget fallback returns it alone, so its AAD covers no other entry; -/// it commits to the chain's heads inside its plaintext instead. +/// the budget fallback returns it alone, so its AAD covers no other entry: +/// it commits to nothing beneath it, and the chain's heads (which chain from +/// nothing) are each checked only as the node at the top of a read. pub(crate) fn no_children() -> ChildrenDigest { children_digest(std::iter::empty()) } diff --git a/packages/stack-encrypt/src/json/derive.rs b/packages/stack-encrypt/src/json/derive.rs index eb1124f67..bcc1e0a90 100644 --- a/packages/stack-encrypt/src/json/derive.rs +++ b/packages/stack-encrypt/src/json/derive.rs @@ -1182,7 +1182,7 @@ fn open_document<'a, K>( let aes = leaf_cipher(&key)?; if open_root { let keyset = responses.next_keyset()?; - open_whole_chain(&doc, &aes, &aad, Selectors::new(keyset, prf_context)) + open_whole_chain(doc, &aes, &aad, Selectors::new(keyset, prf_context)) } else { open_sealed_root(&doc, &aes, &aad) } @@ -1209,7 +1209,7 @@ fn open_sealed_root( /// The whole document with its root open: every entry placed by the chain /// from the root's token, as one read of `$` that must reach every entry. fn open_whole_chain( - doc: &JsonDocument, + doc: JsonDocument, aes: &Aes256Cipher, aad: &Context<'_>, selectors: Selectors, @@ -1233,12 +1233,15 @@ fn open_whole_chain( let _ = open_entry(aes, &doc.header, aad, first, 0, &cascade::no_children())?; return Err(shape("a document whose chain reaches its root")); } - let rows = doc - .entries - .iter() + let JsonDocument { + header, entries, .. + } = doc; + // The entries move into the rows: the document is opened, not kept. + let rows = entries + .into_iter() .enumerate() .map(|(j, entry)| Row { - entry: entry.clone(), + entry, j: j as u64, d: None, }) @@ -1249,7 +1252,7 @@ fn open_whole_chain( probe: false, }]; let opening = Opening { - header: &doc.header, + header: &header, aad, aes, }; @@ -2955,7 +2958,7 @@ mod tests { SafeRand::from_controlled_seed(Protected::new([5; 32])), ) .unwrap(); - open_whole_chain(&doc, &aes, &context, Selectors::for_tests()) + open_whole_chain(doc, &aes, &context, Selectors::for_tests()) } #[test] @@ -3015,7 +3018,7 @@ mod tests { let other = Context::from_encoded(b"users/bio"); let selectors = Selectors::new(Arc::clone(&Selectors::for_tests().keyset), other.clone()); assert!(matches!( - open_whole_chain(&doc, &aes, &other, selectors), + open_whole_chain(doc, &aes, &other, selectors), Err(Error::Aead) )); } diff --git a/packages/stack-encrypt/src/json/query.rs b/packages/stack-encrypt/src/json/query.rs index 1b5fd0b3f..5f3c87f15 100644 --- a/packages/stack-encrypt/src/json/query.rs +++ b/packages/stack-encrypt/src/json/query.rs @@ -24,7 +24,7 @@ use crate::target::{DeclaredContext, Index, Json, Owned, Pending}; use crate::{Error, KeysetCipher, Label, NonEmpty}; /// One selector pair of a projection. -#[derive(Clone, Debug, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] struct Pair16 { s: Tag, vs: Tag, @@ -35,13 +35,47 @@ struct Pair16 { /// element that is a container. What a stored document carries as `q`, /// which the server reads (`json_tree_query`), and what a containment /// needle is. -#[derive(Clone, Debug, PartialEq, Eq, Default)] +/// +/// Equality is as sets, which is how `jsonb @>` reads a projection: two +/// projections are equal when they hold the same pairs in the same groups, +/// whatever order their lists are in. A stored `q` is shuffled, so two +/// seals of one document carry equal projections in different orders. +#[derive(Clone, Debug, Default)] pub struct JsonProjection { sv: Vec, el: Vec, } +impl PartialEq for JsonProjection { + fn eq(&self, other: &Self) -> bool { + Self::cmp_canonical(&self.canonical(), &other.canonical()).is_eq() + } +} +impl std::cmp::Eq for JsonProjection {} + impl JsonProjection { + /// This projection with every list sorted, recursively: one spelling per + /// set of pairs and groups. + fn canonical(&self) -> Self { + let mut sv = self.sv.clone(); + sv.sort_unstable(); + let mut el: Vec = self.el.iter().map(Self::canonical).collect(); + el.sort_unstable_by(Self::cmp_canonical); + Self { sv, el } + } + + /// The order of two projections already in [`canonical`](Self::canonical) + /// form: by their pairs, then by their groups in turn. + fn cmp_canonical(a: &Self, b: &Self) -> std::cmp::Ordering { + a.sv.cmp(&b.sv).then_with(|| { + a.el.iter() + .zip(&b.el) + .map(|(x, y)| Self::cmp_canonical(x, y)) + .find(|o| o.is_ne()) + .unwrap_or_else(|| a.el.len().cmp(&b.el.len())) + }) + } + /// Put the pairs of every group, and the groups of every element list, /// in a uniformly random order ([`order::shuffle`](super::order::shuffle)). /// The server reads `q` as sets (`jsonb @>` over `sv` and `el`), so the @@ -809,6 +843,60 @@ impl Json { #[cfg(test)] mod tests { use super::*; + + fn pair(n: u8) -> Pair16 { + Pair16 { + s: [n; 16], + vs: [n + 100; 16], + } + } + + /// Projections compare as the sets `jsonb @>` reads: the order of their + /// lists does not count, their contents and their grouping do. + #[test] + fn projections_are_equal_as_sets() { + let group = |sv: Vec, el: Vec| JsonProjection { sv, el }; + let a = group( + vec![pair(1), pair(2)], + vec![ + group(vec![pair(3), pair(4)], vec![]), + group(vec![pair(5)], vec![]), + ], + ); + let shuffled = group( + vec![pair(2), pair(1)], + vec![ + group(vec![pair(5)], vec![]), + group(vec![pair(4), pair(3)], vec![]), + ], + ); + assert_eq!(a, shuffled, "the same pairs in the same groups"); + let regrouped = group( + vec![pair(1), pair(2)], + vec![ + group(vec![pair(3)], vec![]), + group(vec![pair(4), pair(5)], vec![]), + ], + ); + assert_ne!(a, regrouped, "the grouping counts"); + let fewer = group(vec![pair(1)], a.el.clone()); + assert_ne!(a, fewer, "a pair fewer"); + let more_groups = group(a.sv.clone(), { + let mut el = a.el.clone(); + el.push(JsonProjection::default()); + el + }); + assert_ne!(a, more_groups, "a group more"); + let nested = group( + vec![], + vec![group(vec![], vec![group(vec![pair(1)], vec![])])], + ); + assert_ne!( + nested, + group(vec![], vec![group(vec![pair(1)], vec![])]), + "depth counts" + ); + } use serde_json::json; #[test] diff --git a/packages/stack-encrypt/src/target/pending.rs b/packages/stack-encrypt/src/target/pending.rs index ebdd2523d..116c90029 100644 --- a/packages/stack-encrypt/src/target/pending.rs +++ b/packages/stack-encrypt/src/target/pending.rs @@ -14,11 +14,13 @@ use std::borrow::Cow; use std::collections::HashMap; use std::future::{Future, IntoFuture}; use std::pin::Pin; +use std::sync::Arc; use stack_kms::{DataKey, DataKeySource, GenerateKeyPayload, Iv, MaybeSend, RetrieveKeyPayload}; use uuid::Uuid; use super::request::{tally, Request, RequestKind, Responses}; +use crate::keyset::KeysetState; use crate::{Descriptor, Error, KeysetCipher, StackCipher}; /// The boxed fulfilment: consumes this pending's slice of the responses and @@ -577,10 +579,21 @@ async fn dispatch( .collect::>()?; // A keyset request is answered from the cipher's cache, and loaded from - // ZeroKMS only on a miss, as selecting the keyset would load it. + // ZeroKMS only on a miss, as selecting the keyset would load it. Each + // keyset is looked up once however many requests name it (a column of + // documents names one), and the answer is shared in request order. + let mut loaded: HashMap> = HashMap::new(); let mut keysets = Vec::with_capacity(loads.len()); for keyset_id in loads { - keysets.push(cipher.keyset_state(keyset_id).await?); + let state = match loaded.get(&keyset_id) { + Some(state) => Arc::clone(state), + None => { + let state = cipher.keyset_state(keyset_id).await?; + let _ = loaded.insert(keyset_id, Arc::clone(&state)); + state + } + }; + keysets.push(state); } Ok(Responses::new(generated, retrieved).with_keysets(keysets)) @@ -612,6 +625,10 @@ mod tests { /// The keyset each call named, in call order. generate_keysets: Mutex>>, retrieve_keysets: Mutex>>, + /// Index-key loads, and an id every load answers with instead of the + /// one asked for, when set. + load_calls: AtomicUsize, + answers_as: Mutex>, } impl CountingSource { @@ -638,6 +655,10 @@ mod tests { fn retrieve_keysets(&self) -> Vec> { self.retrieve_keysets.lock().unwrap().clone() } + + fn load_calls(&self) -> usize { + self.load_calls.load(Ordering::Relaxed) + } } impl DataKeySource for CountingSource { @@ -681,10 +702,77 @@ mod tests { &self, keyset_id: Option, ) -> Result<(Uuid, IndexKey), stack_kms::Error> { - self.inner.load_index_key(keyset_id).await + self.load_calls.fetch_add(1, Ordering::Relaxed); + let (id, key) = self.inner.load_index_key(keyset_id).await?; + Ok((self.answers_as.lock().unwrap().unwrap_or(id), key)) } } + /// A pending that asks for the loaded state of `keyset` and resolves to + /// its id. + fn loading( + cipher: &StackCipher, + keyset: Uuid, + ) -> Pending<'_, Uuid, CountingSource> { + Pending::request(cipher, vec![Request::load_keyset(keyset)], |responses| { + Ok(responses.next_keyset()?.id) + }) + } + + /// Keyset requests are answered once per keyset however many pendings + /// ask, from ZeroKMS on a miss and from the cache after. + #[tokio::test] + async fn keyset_requests_load_each_keyset_once_and_then_hit_the_cache() { + let cipher = cipher().await; + let at_init = cipher.kms().load_calls(); + let other = FakeDataKeySource::new() + .load_index_key(Some(IdentifiedBy::Uuid(Uuid::from_u128(42)))) + .await + .unwrap() + .0; + let pair = loading(&cipher, other) + .zip(loading(&cipher, other)) + .await + .unwrap(); + assert_eq!( + pair, + (other, other), + "both answered with the keyset asked for" + ); + assert_eq!( + cipher.kms().load_calls(), + at_init + 1, + "one load for two requests" + ); + assert_eq!(loading(&cipher, other).await.unwrap(), other); + assert_eq!(cipher.kms().load_calls(), at_init + 1, "then the cache"); + let default = cipher.default_keyset().keyset_id(); + assert_eq!(loading(&cipher, default).await.unwrap(), default); + assert_eq!( + cipher.kms().load_calls(), + at_init + 1, + "the default is never loaded again" + ); + } + + /// A source that answers a load by id for another keyset is refused, + /// and nothing is cached under either id. + #[tokio::test] + async fn a_keyset_answered_for_another_id_is_refused() { + let cipher = cipher().await; + let asked = Uuid::from_u128(42); + let answered = Uuid::from_u128(43); + *cipher.kms().answers_as.lock().unwrap() = Some(answered); + let refused = loading(&cipher, asked).await; + assert!( + matches!(refused, Err(Error::KeysetMismatch { left, right }) if left == asked && right == answered), + "{refused:?}" + ); + let loads = cipher.kms().load_calls(); + let _ = loading(&cipher, asked).await; + assert_eq!(cipher.kms().load_calls(), loads + 1, "nothing was cached"); + } + fn d() -> Descriptor { Descriptor::of("test/field") } From 5bafdfcf544e99a246914bc0b1cb5dfcec2a39db Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 13:56:48 +1100 Subject: [PATCH 08/22] docs(eql): stop promising document order from the json_tree walks Three doc comments in functions.sql said rows come back in document order. Under the random stored order the path query returns entries in stored order, which says nothing about the document; the children walk returns them by ordinal, which is position for an array and keyed rank for an object; and the subtree walk promises no order at all, since the client places every row by its tags. Comments only; no function changes. Refs #1357. Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- packages/eql/src/v3/json_tree/functions.sql | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/packages/eql/src/v3/json_tree/functions.sql b/packages/eql/src/v3/json_tree/functions.sql index 5951aa0e7..c3e51b88b 100644 --- a/packages/eql/src/v3/json_tree/functions.sql +++ b/packages/eql/src/v3/json_tree/functions.sql @@ -293,7 +293,9 @@ $$; --! @param val jsonb The document. --! @param selector text The path selector, base64. --! @return SETOF public.{{prefix}}_json_tree_entry The matching entries, in ---! document order. +--! stored order, which says nothing about the document: entries +--! are stored in a random order, and the client puts an array's +--! elements back in order from their decrypted labels. CREATE FUNCTION {{prefix}}.json_tree_path_query(val jsonb, selector text) RETURNS SETOF public.{{prefix}}_json_tree_entry LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE @@ -520,7 +522,8 @@ $$; --! @brief The children of the node whose token is `token`, as rows. --! --! What `jsonb_array_elements` is for an encrypted array, and the members of ---! an encrypted object: one row per child, in the document's order, each +--! an encrypted object: one row per child, in ordinal order (an array's +--! elements by position, an object's members by their keyed rank), each --! opening on its own. Enumerating one level needs no child's token, so the --! rows are the same whichever rule each child's token takes. --! @@ -634,9 +637,10 @@ $$; --! miss and recursing under each hit's own token. So a read returns the --! node it names, and `{}`, `[]`, a scalar and an absent node are an empty --! object's entry, an empty array's entry, the scalar's entry and no rows, ---! the four answers `jsonb #>` gives. Rows come out in token order and, ---! within a token, in document order. A token that names no node yields no ---! row, and nothing in a row is linkable to anything without its token. +--! the four answers `jsonb #>` gives. Rows come out in no promised order: +--! the client places each one by its tags and checks it under its parent's +--! digest, so the order they arrive in carries nothing. A token that names +--! no node yields no row, and nothing in a row is linkable to anything without its token. --! The client sends one token for a path with no wildcard, and for a --! wildcard one per position below its bound and a probe at the bound. The --! server treats a probe like any other token, but can pick it out: it is From 810852c65ed3c95cb699c3e0412029be13d5db36 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 15:24:39 +1100 Subject: [PATCH 09/22] feat(stack-encrypt): send every JSON needle in a random order of its own With the stored order random, the client's containment and equality needles were the last thing in key order: built in the walk's order, an object's members in their keys' byte order, so a query with two sibling keys told the database which sorts first, at selectors it also sees in the stored rows. The leakage model found it. `JsonQuery::of_terms` and `at_path` now shuffle the needle's projection as a stored `q` is shuffled: every pair list and element-group list in a uniformly random order, recursively, with a fresh permutation per query. The seed is drawn from the cipher's OrderSource when the query's terms are derived (`terms_pending`), so it is the operating system's entropy outside tests and the seeded stream under `test-support`, and it travels in `JsonTerms` until the needle is built. `jsonb @>` is set containment over arrays, so no answer changes; `JsonQuery` equality is already as sets, and two queries for one value now serialize differently and compare equal. Nothing in SQL, the header or the data plan changes. Tests: two needles for one value under the entropy source differ in order and are equal as sets; a proptest over the walk's projections shows a shuffled needle matches exactly the stored projections (in either order) the walk-order needle matched; the generated containment property derives every needle twice and holds both to `jsonb @>`. The PostgreSQL json_tree oracle passes unchanged. The leakage model gains a `SentOrder` clause; a mutant claiming needles go in the walk's order is caught by `repeated_needles_sent_alike`, and the key-order reading test now asserts that nothing the database sees tells the two readings apart. The leakage page, the design note and the changeset say so. Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- .../eql-json-tree-cascade-and-random-order.md | 2 +- docs/plans/2026-10-17-json-tree-index.md | 4 +- .../eql/docs/reference/json-tree-leakage.md | 2 +- packages/stack-encrypt/src/json/derive.rs | 54 ++++++++++++- packages/stack-encrypt/src/json/order.rs | 19 +++-- packages/stack-encrypt/src/json/query.rs | 25 +++++- packages/stack-encrypt/tests/json_index.rs | 55 +++++++++++++ .../tests/json_leakage/calibration.rs | 53 +++++++----- .../tests/json_leakage/canonical.rs | 72 ++++++++++------- .../tests/json_leakage/corpus.rs | 12 ++- .../tests/json_leakage/distinguishers.rs | 78 +++++++++++++++--- .../stack-encrypt/tests/json_leakage/mod.rs | 80 +++++++++---------- .../tests/json_leakage/mutants.rs | 24 +++--- .../tests/json_leakage/observe.rs | 16 +++- .../tests/json_leakage/profile.rs | 56 +++++++++---- .../tests/json_leakage/simulator.rs | 23 +++--- 16 files changed, 419 insertions(+), 156 deletions(-) diff --git a/.changeset/eql-json-tree-cascade-and-random-order.md b/.changeset/eql-json-tree-cascade-and-random-order.md index f38529834..8e76e57fa 100644 --- a/.changeset/eql-json-tree-cascade-and-random-order.md +++ b/.changeset/eql-json-tree-cascade-and-random-order.md @@ -2,4 +2,4 @@ "@cipherstash/eql": patch --- -EQL v4 `json_tree` (unreleased): a stored document's entries are now in a random order, root first, and its containment projection `q` is shuffled the same way, so the stored order no longer shows each row's tree, the size of every subtree or the byte order of sibling key names (#1357). Every entry's ciphertext now also commits to its children's AEAD tags, so the client checks a subtree read's rows by the chain rather than by their positions, and a row swapped, dropped, added, reordered or moved beneath the node read is refused by the cipher. Nothing stored changes shape and no SQL changes; documents sealed by the earlier build of this unreleased family no longer open. `->`, `->>` and an ordering on `eql_v4.ord_term(col -> sel)` at a wildcard path now return an unspecified element of the row (the same one every time), not element 0; read every element with `json_tree_path_query`. `docs/reference/json-tree-leakage.md` says what the stored row still shows and what a read now proves. +EQL v4 `json_tree` (unreleased): a stored document's entries are now in a random order, root first, and its containment projection `q` is shuffled the same way, so the stored order no longer shows each row's tree, the size of every subtree or the byte order of sibling key names (#1357). A containment or equality needle is sent in a random order of its own too, so a query no longer shows sibling key order either. Every entry's ciphertext now also commits to its children's AEAD tags, so the client checks a subtree read's rows by the chain rather than by their positions, and a row swapped, dropped, added, reordered or moved beneath the node read is refused by the cipher. Nothing stored changes shape and no SQL function changes (three doc comments stop promising document order); documents sealed by the earlier build of this unreleased family no longer open. `->`, `->>` and an ordering on `eql_v4.ord_term(col -> sel)` at a wildcard path now return an unspecified element of the row (the same one every time), not element 0; read every element with `json_tree_path_query`. `docs/reference/json-tree-leakage.md` says what the stored row still shows and what a read now proves. diff --git a/docs/plans/2026-10-17-json-tree-index.md b/docs/plans/2026-10-17-json-tree-index.md index 8f043cca1..e25a1d552 100644 --- a/docs/plans/2026-10-17-json-tree-index.md +++ b/docs/plans/2026-10-17-json-tree-index.md @@ -712,10 +712,10 @@ The stored order was the last disclosure of the format that no option changed: e 2. **A random stored order is the format.** Not an option, a flag or a switch. The root stays at slot 0, where the sealed-root fallback (`sv -> 0`) finds it; every other entry is placed by a uniformly random permutation of the slots 1…n, and `q`'s pair lists and element groups are shuffled the same way. The permutation is drawn per document at sealing time and thrown away: it is derived from nothing, keyed or not, because nobody needs to recompute it, and a permutation derived from anything stored (the `iv`) would be computable by the database. The generator is vitaminc's `SafeRand::from_entropy` with `next_below` (Lemire's reduction, bias at most n / 2⁶⁴) driving Fisher–Yates (`packages/stack-encrypt/src/json/order.rs`); it is not part of the format, since nothing stored depends on it. For fixtures, `StackCipherBuilder::json_order_seed` (behind `test-support`) draws every document's order from one stream started at a seed, through `SafeRand::from_controlled_seed`, and the deterministic test cipher sets it: `tests/fixtures/json_tree_vectors.json` now pins, per root option, the entries in their stored order, the path stored at each slot, the selector multiset, `q` as sets, and the tokens. The tags and tokens did not move. - What stays visible at rest, and no order hides: the multiset of selectors (paths, node counts, array lengths, whether two rows have the same paths in the same numbers), `q`'s grouping by array element, which entries are heads under the sealed root (the head count, no longer the size of each head's subtree), each ciphertext's length, and the ordering terms at ordered paths. The leakage page drops its stored-order row and says this; the executable profile in `tests/json_leakage` models the new order, reading every world in an order an observer can rebuild without the stored one (`canonical.rs`). One disclosure of the key order remains, at query time: a containment or equality needle is still sent in the walk's order, members in key byte order, so a needle with two sibling keys shows which sorts first. The page says so; shuffling needles on the client (the server reads them as sets) would close it, and is left for a decision of its own. + What stays visible at rest, and no order hides: the multiset of selectors (paths, node counts, array lengths, whether two rows have the same paths in the same numbers), `q`'s grouping by array element, which entries are heads under the sealed root (the head count, no longer the size of each head's subtree), each ciphertext's length, and the ordering terms at ordered paths. The leakage page drops its stored-order row and says this; the executable profile in `tests/json_leakage` models the new order, reading every world in an order an observer can rebuild without the stored one (`canonical.rs`). A containment or equality needle is shuffled the same way, with a fresh permutation per query drawn from the same `OrderSource` (`JsonQuery::of_terms` and `at_path`; the seed travels in the query's `JsonTerms`), so the query-time profile matches the at-rest one: in the walk's order a needle would show which of two sibling keys sorts first. `jsonb @>` is set containment over arrays, so the order changes no answer, and `JsonProjection` (and so `JsonQuery`) compares as sets. Added at the owner's request after review, in the same PR. 3. **First match at a wildcard path is unspecified (owner, 2026-10-26).** `json_tree_path_query_first`, and so `->`, `->>` and an ordering on `eql_v4.ord_term(col -> sel)`, returns the first entry in `sv` carrying a selector. At a wildcard path (every element under `$.a[*]`, at any depth) that is now an unspecified element, stable for a stored row but not element 0. Ordering or comparing over the elements of a wildcard path is written over all the matches (`json_tree_path_query`, `EXISTS` or `ORDER BY` over their ordering terms), and element order within a document is recovered after decryption from the labels, as it already was. No SQL changes. Two variants were considered and dropped by the owner: a fix-up after the shuffle that kept each wildcard selector's document-first entry first (it would have revealed, per wildcard selector, which entry is the document-first one), and writing the heads' digest into the sealed root's plaintext so that a client reading the root with its heads could verify them. The sealed root commits to nothing beneath it. 4. **The keyset is loaded on a root-open decrypt.** Opening a whole document whose root is open walks the chain from the root's token, which needs the index key of the keyset it was sealed under. The open asks for it beside the data key, as a new kind of request a `Pending` carries (`Request::load_keyset`), answered from the cipher's keyset cache or loaded from ZeroKMS on a miss, as `StackCipher::keyset` loads one. Settling a pending still asks only for a `DataKeySource`: the cipher captures the backend's `load_index_key` at `init`, where `IndexKeySource` is known. The sealed root (the default) loads nothing. -Left as it was, and worth a follow-up: two SQL doc comments in `src/v3/json_tree/functions.sql` say rows come back "in document order" (`json_tree_path_query`, and `json_tree_subtree` within a token). Both return rows by slot (`ORDER BY` on `j`), which is now the random stored order; no caller depends on it, and the client places rows by the chain. They are left for a change that may touch SQL. The benchmarks were not re-run: nothing stored and nothing in SQL changed. The client's added work is one SHA-256 per container over its children's tags, at sealing and at opening, and at opening the parent-tag probes the server already runs, which replace the position checks. +The SQL doc comments that promised "document order" (`json_tree_path_query`, `json_tree_children`, `json_tree_subtree`) now say what each returns: the stored order, ordinal order, and no promised order; no function changed. The benchmarks were not re-run: nothing stored and no SQL function changed. The client's added work is one SHA-256 per container over its children's tags, at sealing and at opening, and at opening the parent-tag probes the server already runs, which replace the position checks. diff --git a/packages/eql/docs/reference/json-tree-leakage.md b/packages/eql/docs/reference/json-tree-leakage.md index 6b248e88c..4fe0f5122 100644 --- a/packages/eql/docs/reference/json-tree-leakage.md +++ b/packages/eql/docs/reference/json-tree-leakage.md @@ -42,7 +42,7 @@ From these the tree can still be partly inferred across rows: a path never occur A query hands the database a term derived from the plaintext it asks about, and the database's answer is visible to it. Each query type reveals the following, in addition to the rows it returns. -- **Containment (`@>`, `<@`), and equality at a path, which is sent as a containment needle.** The needle's own projection: the selectors of its paths, the value selectors of its nodes, and its element grouping. So the database learns which paths and which (path, value) pairs you asked about, and which rows hold them. Unlike the stored projection, a needle is sent in the order the client walks it, an object's members in the byte order of their key names, so a needle with two sibling keys shows which of them sorts first, at selectors the database also sees in the stored rows. +- **Containment (`@>`, `<@`), and equality at a path, which is sent as a containment needle.** The needle's own projection: the selectors of its paths, the value selectors of its nodes, and its element grouping. So the database learns which paths and which (path, value) pairs you asked about, and which rows hold them. A needle is sent in a random order too, drawn afresh for each query, so it shows no more of the order of the needle's keys than a stored row does. - **Path extraction (`->`, `->>`, `json_tree_path_query`, `json_tree_path_exists`).** The selector of the path asked. A position in the path reads as the wildcard, so a position is not sent. The entry returned is a ciphertext. At a wildcard path (any path with an array step, at any depth: every element under `$.orders[*]`), `->`, `->>` and an ordering on `eql_v4.ord_term(col -> sel)` take the first entry carrying the selector in the stored order, which is random per row: an unspecified element, the same one every time that row is read, and not element 0. To compare, order or read every element, use the all-match form (`json_tree_path_query`, or `EXISTS` and `ORDER BY` over the matches' ordering terms); a client recovers element order after decryption, from each element's position, which is inside its ciphertext. - **Ordering at a path (`<`, `<=`, `>`, `>=`).** The path's selector and the bound's own ordering term, which compares with the stored terms exactly as they compare with each other: the database learns the bound's place in the order of the stored values at every ordered path, and whether it equals one. `ORDER BY ord_term(…)` sends no term and shows nothing the stored terms do not. - **A subtree read (`json_tree_subtree`, `json_tree_children`, `json_tree_child_count`).** A **token** for exactly the node the path names, which the client derives from the path alone. Reading `$.profile.address.city` sends that leaf's token and nothing that covers `$.profile`. No token over a top-level field or an array leaves the client unless that field or array is itself what is read. diff --git a/packages/stack-encrypt/src/json/derive.rs b/packages/stack-encrypt/src/json/derive.rs index bcc1e0a90..0838decef 100644 --- a/packages/stack-encrypt/src/json/derive.rs +++ b/packages/stack-encrypt/src/json/derive.rs @@ -217,6 +217,12 @@ pub struct JsonTerms { /// scalar has as an element of a root array, so a containment needle /// can match a stored `["a"]` as `jsonb @>` does. alt: Option<(Tag, Tag)>, + /// The seed of the order a needle built from these terms is sent in, + /// drawn from the cipher's [`OrderSource`](super::order::OrderSource) + /// when the terms are derived for a query ([`terms_pending`]); `None` + /// for the terms a document is sealed from, which no needle is built + /// from. A seed decides a permutation of public pairs and nothing else. + needle_order: Option>, } /// The key names and the tokens are what the terms must not leak, so the @@ -258,6 +264,14 @@ impl JsonTerms { self.alt } + /// The generator a needle built from these terms is shuffled with, if + /// the terms were derived for a query. + pub(crate) fn needle_order(&self) -> Option { + self.needle_order + .clone() + .map(SafeRand::from_controlled_seed) + } + /// The last node of the walk: in a needle built from one path, the /// leaf the path names. pub(crate) fn leaf(&self) -> Option<&TermNode> { @@ -587,6 +601,7 @@ fn walk( terms: JsonTerms { nodes: walker.nodes, alt, + needle_order: None, }, plain: walker.plain, }) @@ -749,8 +764,17 @@ pub(crate) fn terms_pending<'a, K>( ) -> Pending<'a, JsonTerms, K> { let context = context.into_prf_context().into_owned(); let terms = walk(value, options, cipher.prf(), &context) - .map(|w| w.terms) - .map_err(Error::from); + .map_err(Error::from) + .and_then(|w| { + // A needle is sent in a random order of its own, drawn as a + // document's stored order is ([`order`]): fresh entropy per + // query outside tests. + let mut order = cipher.cipher().json_order().document()?; + Ok(JsonTerms { + needle_order: Some(order::seed(&mut order)), + ..w.terms + }) + }); Pending::ready(cipher, terms) } @@ -2598,6 +2622,32 @@ mod tests { } } + proptest! { + /// A needle sent in a shuffled order matches exactly the stored + /// projections the needle in the walk's order matched, stored + /// in either order: `jsonb @>` is set containment over arrays. + #[test] + fn a_shuffled_needle_matches_what_the_unshuffled_one_did( + stored in arbitrary_json(), + needle in arbitrary_json(), + seed in any::<[u8; 32]>(), + ) { + let options = JsonOptions::default(); + let project = |value: &Value| super::super::super::query::group_of(terms(value, &options).nodes(), None); + let mut rng = SafeRand::from_controlled_seed(Protected::new(seed)); + let walked = project(&needle); + let mut shuffled = walked.clone(); + shuffled.shuffle(&mut rng); + let mut stored_shuffled = project(&stored); + stored_shuffled.shuffle(&mut rng); + prop_assert_eq!(&shuffled, &walked, "one needle, as sets"); + for target in [project(&stored), stored_shuffled] { + prop_assert_eq!(target.contains(&shuffled), target.contains(&walked)); + } + prop_assert!(project(&needle).contains(&shuffled), "a needle contains itself in any order"); + } + } + /// Small documents over the keys that stress the path encoding. pub(super) fn arbitrary_json() -> impl Strategy { let leaf = prop_oneof![ diff --git a/packages/stack-encrypt/src/json/order.rs b/packages/stack-encrypt/src/json/order.rs index b72491a9e..5554a27a4 100644 --- a/packages/stack-encrypt/src/json/order.rs +++ b/packages/stack-encrypt/src/json/order.rs @@ -65,17 +65,24 @@ impl OrderSource { let mut stream = stream .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); - let seed = Protected::generate(|| { - let mut seed = [0u8; 32]; - stream.fill_bytes(&mut seed); - seed - }); - Ok(SafeRand::from_controlled_seed(seed)) + Ok(SafeRand::from_controlled_seed(seed(&mut stream))) } } } } +/// Thirty-two bytes drawn from `rng`, as the seed of another generator: +/// one document's order from a controlled stream, or the order a query's +/// needle is sent in, which its terms carry until the needle is built +/// ([`JsonTerms`](super::JsonTerms)). +pub(crate) fn seed(rng: &mut SafeRand) -> Protected<[u8; 32]> { + Protected::generate(|| { + let mut seed = [0u8; 32]; + rng.fill_bytes(&mut seed); + seed + }) +} + /// A uniform shuffle of `items` (Fisher–Yates), each swap index drawn with /// [`SafeRand::next_below`]. pub(crate) fn shuffle(items: &mut [T], rng: &mut SafeRand) { diff --git a/packages/stack-encrypt/src/json/query.rs b/packages/stack-encrypt/src/json/query.rs index 5f3c87f15..11796199b 100644 --- a/packages/stack-encrypt/src/json/query.rs +++ b/packages/stack-encrypt/src/json/query.rs @@ -109,6 +109,14 @@ impl JsonProjection { /// stored root array holding it. Serializes to the `query_json_tree` /// operand: `{"sv": [{"s": "…", "vs": "…"}, …], "el": [{"sv": …, "el": …}, /// …], "alt": {…}}`. +/// +/// The projection is sent in a random order of its own, drawn for each +/// query as a stored `q`'s order is drawn for each document: every pair +/// list and element-group list shuffled, recursively. `jsonb @>` is set +/// containment over arrays, so the order changes no answer, and a needle +/// shows no more of the order of its keys than a stored row does. Two +/// queries for one value therefore serialize differently and compare equal +/// (equality is as sets). #[derive(Clone, Debug, PartialEq, Eq)] pub struct JsonQuery { group: JsonProjection, @@ -122,7 +130,7 @@ impl JsonQuery { /// `("$", 1)` asks for the scalar `1`, not for an array holding it. pub(crate) fn at_path(terms: &JsonTerms) -> Self { Self { - group: group_of(terms.nodes(), None), + group: needle(terms), alt: None, } } @@ -133,7 +141,7 @@ impl JsonQuery { el: Vec::new(), }); Self { - group: group_of(terms.nodes(), None), + group: needle(terms), alt, } } @@ -152,6 +160,19 @@ impl JsonQuery { } } +/// The projection a needle sends: the terms' group, its pair lists and +/// element groups shuffled, recursively, with the generator the terms carry +/// ([`JsonProjection::shuffle`]), as a stored `q` is shuffled. `jsonb @>` +/// is set containment over arrays, so the order changes no answer; in the +/// walk's order a needle would show which of two sibling keys sorts first. +fn needle(terms: &JsonTerms) -> JsonProjection { + let mut group = group_of(terms.nodes(), None); + if let Some(mut order) = terms.needle_order() { + group.shuffle(&mut order); + } + group +} + /// The group of the nodes that sit directly in `parent` (`None` is the top /// level), with the container elements in it nested as groups. What a /// needle is, and what a document writes to `q` when it is sealed. diff --git a/packages/stack-encrypt/tests/json_index.rs b/packages/stack-encrypt/tests/json_index.rs index ec3b5d3b9..00ac923bb 100644 --- a/packages/stack-encrypt/tests/json_index.rs +++ b/packages/stack-encrypt/tests/json_index.rs @@ -1744,6 +1744,41 @@ mod given_the_design_corpus { .collect() } + /// A needle is sent in a random order of its own, drawn for each query: + /// two needles for one value, under the operating system's entropy, + /// carry their pairs in different orders and are equal as sets. Twelve + /// top-level pairs, so the two orders agree with probability 1 / 12!. + #[tokio::test] + async fn sends_each_needle_in_an_order_of_its_own() { + let Scenario { cipher, plan } = scenario().await; + let value: Value = (0..11) + .map(|i| (format!("k{i}"), json!(i))) + .collect::>() + .into(); + let one: JsonQuery = cipher.query(op::Contains, &value).on(&plan).await.unwrap(); + let two: JsonQuery = cipher.query(op::Contains, &value).on(&plan).await.unwrap(); + let (a, b) = ( + serde_json::to_value(&one).unwrap(), + serde_json::to_value(&two).unwrap(), + ); + assert_eq!( + a["sv"].as_array().unwrap().len(), + 12, + "the root and eleven members" + ); + assert_ne!(a, b, "two orders"); + assert_eq!(as_set(&a), as_set(&b), "one set"); + assert_eq!(one, two, "equal as sets"); + let at: JsonQuery = cipher + .query(op::AtPath(op::Eq), &("$.k3", 3)) + .on(&plan) + .await + .unwrap(); + let doc = cipher.encrypt(&value).using(&plan).await.unwrap(); + assert!(one.matches(doc.projection()) && two.matches(doc.projection())); + assert!(at.matches(doc.projection()), "an equality needle too"); + } + /// Every pair of the corpus, both ways round: `@>` over the projections /// is `jsonb @>`, and `<@` is `jsonb <@` but for a stored root scalar in /// a needle root array. @@ -2186,6 +2221,26 @@ mod given_generated_documents { &case.stored, &needle ); + // Every query sends its needle in a random order of its own: a + // second derivation is the same set of pairs and groups, and + // matches exactly what the first did. + let again: JsonQuery = runtime + .block_on( + scenario + .cipher + .query(op::Contains, &needle) + .on(&scenario.plan) + .into_future(), + ) + .unwrap(); + prop_assert_eq!(&again, &query, "one needle, as sets"); + prop_assert_eq!( + again.matches(doc.projection()), + query.matches(doc.projection()), + "{} @> {}, sent in another order", + &case.stored, + &needle + ); Ok(()) }); } diff --git a/packages/stack-encrypt/tests/json_leakage/calibration.rs b/packages/stack-encrypt/tests/json_leakage/calibration.rs index d33599f6c..a79917be8 100644 --- a/packages/stack-encrypt/tests/json_leakage/calibration.rs +++ b/packages/stack-encrypt/tests/json_leakage/calibration.rs @@ -307,32 +307,45 @@ mod given_integers_ordered_exactly { mod given_the_jsonb_reading_of_the_key_order { use super::*; - /// The stored order no longer shows the key order, but a containment - /// needle is sent in the walk's order, an object's members sorted by - /// their keys' UTF-8 bytes. Read as `jsonb`'s sort (shorter keys first), - /// the profile is told from the real scheme by a needle whose keys sort - /// differently the two ways (`{"a": …, "a b": …, "é": …, "0": …}`): a - /// needle with sibling keys tells the database their byte order, and a - /// reader inferring key names from it needs to know which sort it is. + /// The page once had to say which sort "the order of their key names" + /// meant, because the stored order showed it, and then a needle did. Now + /// neither does: the entries are stored in a random order, a needle is + /// sent in one, and a member's ordinal is a keyed rank. So the two + /// readings, the keys' UTF-8 bytes and `jsonb`'s (shorter keys first), + /// give different profiles on a column whose keys sort differently the + /// two ways (a row and a needle `{"a": …, "a b": …, "é": …, "0": …}`), + /// and no distinguisher tells either from the real scheme: the key order + /// is nothing the database learns, and the page need not say which. #[test] - fn a_needle_tells_it_apart() { + fn nothing_tells_it_apart() { let runtime = runtime(); let cipher = runtime.block_on(stack_cipher()); - let clauses = Clauses { + let jsonb = Clauses { key_order: KeyOrder::JsonbLengthFirst, ..Clauses::default() }; - let world = &handcrafted()[0]; - let real = real_world(&runtime, &cipher, world); - let leakage = leak(&world.column, clauses, &world.docs, &world.queries); - let fired: Vec<&str> = separating(&features(&real), &leakage, &mut rng()) - .into_iter() - .map(|(n, _, _)| n) - .collect(); - assert!( - fired.contains(&"transcripts"), - "the order of a needle's pairs tells byte order from jsonb order: {fired:?}" - ); + for world in &handcrafted()[..2] { + let real = real_world(&runtime, &cipher, world); + let bytes = leak( + &world.column, + Clauses::default(), + &world.docs, + &world.queries, + ); + let leakage = leak(&world.column, jsonb, &world.docs, &world.queries); + assert_ne!( + leakage, bytes, + "{}: the two readings differ in the profile", + world.name + ); + let separated = separating(&features(&real), &leakage, &mut rng()); + assert!( + separated.is_empty(), + "{}: the key order shows after all:\n{}", + world.name, + report(&separated) + ); + } } } diff --git a/packages/stack-encrypt/tests/json_leakage/canonical.rs b/packages/stack-encrypt/tests/json_leakage/canonical.rs index b1cb418a0..4bd0f0d94 100644 --- a/packages/stack-encrypt/tests/json_leakage/canonical.rs +++ b/packages/stack-encrypt/tests/json_leakage/canonical.rs @@ -1,14 +1,16 @@ -//! The canonical order: each stored row's entries and every list of its -//! projection `q`, put in an order an observer computes from the column and -//! the queries without the stored order, so that the distinguishers can -//! name an entry by its place and still compare a real world with a -//! simulated one for equality. +//! The canonical order: each stored row's entries, every list of its +//! projection `q` and every list of each needle sent, put in an order an +//! observer computes from the column and the queries without the stored or +//! sent order, so that the distinguishers can name an entry by its place +//! and still compare a real world with a simulated one for equality. //! -//! The stored order is random ([`super::profile::StoredOrder::Random`]), in -//! the real scheme and in a faithful simulation alike, independently. A -//! feature that named an entry by its slot, or a selector by the slot it -//! first appears at, would tell the two apart by chance alone. So every -//! distinguisher reads the world in this order instead: +//! The stored order is random ([`super::profile::StoredOrder::Random`]), and +//! so is the order a needle is sent in +//! ([`super::profile::SentOrder::Random`]), in the real scheme and in a +//! faithful simulation alike, independently. A feature that named an entry +//! by its slot, or a selector by the slot it first appears at, or a needle's +//! pairs in the order they came, would tell the two apart by chance alone. +//! So every distinguisher reads the world in this order instead: //! //! - Every name (a selector or a value selector) gets a rank from what is //! attached to it, in tiers compared in turn. At rest: a selector by, per @@ -18,10 +20,10 @@ //! depths it is sent at); a value selector by that colour. Then, //! repeatedly until nothing more splits, the groups of `q` a name's pairs //! sit in, by the ranks of their pairs, of their parent group's and of -//! their element groups'. Then the queries that name it (a needle's pairs -//! by their place in it, since a needle is sent in the walk's order), -//! then where walks reach a selector's entries, then the needles' groups -//! as `q`'s were. Names still tied are ones nothing tells apart (`$.a.b` +//! their element groups'. Then the queries that name it (an extraction, +//! an ordering bound, or a needle and the depth it sits at there), then +//! where walks reach a selector's entries, then the needles' groups as +//! `q`'s were. Names still tied are ones nothing tells apart (`$.a.b` //! and `$.a.b.c` in a row `{"a": {"b": {"c": {}}}}`, at rest); they are //! set apart one at a time, each followed by the refinement again, so //! that one choice orders the entries and `q` alike. @@ -35,8 +37,8 @@ //! down what the walk shows beneath it and, for an array's elements, the //! ordinal (an object's members' ordinals are a keyed rank, a random //! permutation that would order them differently in every world). -//! - Every list of `q` is sorted by its pairs' ranks, and its element -//! groups by their sorted contents. +//! - Every list of `q`, and of every needle and `alt`, is sorted by its +//! pairs' ranks, and its element groups by their sorted contents. //! //! Two entries the sort ties agree on everything a distinguisher reads of //! them but their random bytes (a ciphertext, a self tag or a parent tag no @@ -49,11 +51,11 @@ //! differ (a mutant's) reorders only entries that agree at rest, and a //! named node's place does not move with what the walks below it reach. //! -//! Needles are sent in the walk's order, which is part of what they reveal -//! ([`super::profile::GroupLeakage::pairs`]), and are left as sent. Each -//! entry's stored slot is kept beside it ([`ObservedRow::slots`]) for the -//! feature that reads the stored order itself -//! ([`super::distinguishers::walked_subtrees_are_contiguous`]). +//! Each entry's stored slot is kept beside it ([`ObservedRow::slots`]), and +//! each query as sent ([`ObservedWorld::sent`]), for the two features that +//! read the stored and the sent order themselves +//! ([`super::distinguishers::walked_subtrees_are_contiguous`], +//! [`super::distinguishers::repeated_needles_sent_alike`]). use std::collections::{BTreeMap, BTreeSet, HashMap}; @@ -111,9 +113,24 @@ pub fn canonical(world: &ObservedWorld) -> ObservedWorld { } }) .collect(); + let queries = world + .queries + .iter() + .map(|query| match query { + ObservedQuery::Contains { needle, alt } => ObservedQuery::Contains { + needle: group(needle, &ranks), + alt: alt.as_ref().map(|a| group(a, &ranks)), + }, + ObservedQuery::Equals { needle } => ObservedQuery::Equals { + needle: group(needle, &ranks), + }, + other => other.clone(), + }) + .collect(); ObservedWorld { rows, - queries: world.queries.clone(), + queries, + sent: world.sent.clone(), query_fields: world.query_fields.clone(), } } @@ -510,18 +527,17 @@ fn ranks( _ => {} } } - // A needle is sent in the walk's order, so a pair's place in it is - // part of what the database sees (`{"list": …, "name": …}` sends the - // two in their keys' byte order, which may be all that tells two - // selectors apart). + // A needle is sent in a random order, so only which needle a name is + // in, and at what depth, says anything; the groups it sits in there are + // the refinement's. for (what, needle) in needles(world) { let mut pairs = Vec::new(); pairs_by_depth(needle, 0, &mut pairs); - for (place, (depth, s, vs)) in pairs.into_iter().enumerate() { + for (depth, s, vs) in pairs { for name in [s, vs] { sent.entry(name) .or_default() - .push(format!("{what} {place} {depth}")); + .push(format!("{what} {depth}")); } } } diff --git a/packages/stack-encrypt/tests/json_leakage/corpus.rs b/packages/stack-encrypt/tests/json_leakage/corpus.rs index 274484cfe..f34f1983b 100644 --- a/packages/stack-encrypt/tests/json_leakage/corpus.rs +++ b/packages/stack-encrypt/tests/json_leakage/corpus.rs @@ -74,7 +74,7 @@ fn shape_queries(sealed_root: bool) -> Vec { Query::Contains(json!({"hello": "world-1"})), Query::Contains(json!([{"x": 1}])), // Sibling keys whose byte order is not their order as written, nor - // jsonb's: a needle is sent in the walk's order, so it shows which. + // jsonb's: the needle is shuffled, so it shows neither. Query::Contains(json!({"a": "b", "a b": 1, "é": [], "0": {}})), Query::Equals("$.hello".into(), json!("world-1")), Query::Equals("$.nested.list[*].x".into(), json!(2)), @@ -173,9 +173,11 @@ fn numbers() -> World { /// A column with three wide top-level objects, each read whole, so that /// whether a read's subtree sits at consecutive stored slots is decidable /// rather than a coin toss (`distinguishers::walked_subtrees_are_contiguous`), -/// and an array of twelve like objects, read whole, so that whether their +/// an array of twelve like objects, read whole, so that whether their /// members' ordinals share one order is -/// (`distinguishers::member_order_shared_across_parents`). +/// (`distinguishers::member_order_shared_across_parents`), and a wide needle +/// sent twice, so that whether a needle is sent in one order every time is +/// (`distinguishers::repeated_needles_sent_alike`). fn wide() -> World { let object = |prefix: &str, n: usize| { Value::Object( @@ -205,6 +207,10 @@ fn wide() -> World { Query::subtree("$.w3"), Query::subtree_with("$.arr[*].k", 1, 64), Query::subtree("$.objs"), + // One needle of nineteen pairs, sent twice: shuffled afresh for + // each query, the two sends differ but by a chance of 1 / 19!. + Query::Contains(json!({"w1": object("a", 8), "w3": object("c", 8)})), + Query::Contains(json!({"w1": object("a", 8), "w3": object("c", 8)})), ], } } diff --git a/packages/stack-encrypt/tests/json_leakage/distinguishers.rs b/packages/stack-encrypt/tests/json_leakage/distinguishers.rs index 75f2255ab..9ffef1d23 100644 --- a/packages/stack-encrypt/tests/json_leakage/distinguishers.rs +++ b/packages/stack-encrypt/tests/json_leakage/distinguishers.rs @@ -11,15 +11,18 @@ //! say), which changes a feature with probability below `n² / 2^64` for `n` //! tags in a world, about `10^-13` at the harness's sizes. //! -//! The entries are stored in a random order, so each distinguisher reads a -//! world in its canonical order ([`super::canonical`]): a row's root at -//! position `0` and its other entries, and every list of `q`, sorted by what -//! an observer sees of them without the stored order. A *position* below is -//! a place in that order, and a selector's name is its first appearance in -//! it. One feature reads the stored order itself, through the slot each -//! entry came from ([`walked_subtrees_are_contiguous`]), and like -//! [`member_order_shared_across_parents`] it carries its probability bound -//! in its doc comment. +//! The entries are stored in a random order, and a needle is sent in one, so +//! each distinguisher reads a world in its canonical order +//! ([`super::canonical`]): a row's root at position `0` and its other +//! entries, every list of `q` and every list of a needle, sorted by what an +//! observer sees of them without the stored or sent order. A *position* +//! below is a place in that order, and a selector's name is its first +//! appearance in it. Two features read the orders themselves: the stored +//! one, through the slot each entry came from +//! ([`walked_subtrees_are_contiguous`]), and the sent one, through the +//! queries as sent ([`repeated_needles_sent_alike`]). Like +//! [`member_order_shared_across_parents`] they carry their probability +//! bounds in their doc comments. //! //! What is deliberately not here, and why, is in the module docs //! (`mod.rs`, "Out of reach"). @@ -82,6 +85,7 @@ registry!( token_derivability, subtree_results, transcripts, + repeated_needles_sent_alike, extraction_matches, containment_matches, ordering_term_order, @@ -1219,6 +1223,62 @@ pub fn transcripts(world: &ObservedWorld) -> Vec { .collect() } +/// Whether a needle sent more than once is sent in one order every time: +/// what a needle sent in the walk's order does, and what one shuffled +/// afresh for each query does not. This reads the needles as sent +/// ([`ObservedWorld::sent`]); a needle's identity is its content, read in +/// the canonical order. Reported only where chance cannot answer it. +/// +/// The bound: a needle's pair list of `m` pairs, `c_1, c_2, …` of them +/// alike, is sent in one of `m! / Π c_i!` arrangements, uniformly and +/// independently for each query, and so is its list of `k` element groups, +/// `d_1, d_2, …` of them alike, in one of `k! / Π d_j!`. Two sends of one +/// needle agree everywhere only if those two lists agree, which they do with +/// probability `Π c_i! / m! · Π d_j! / k!` (deeper lists only make it +/// smaller), and `t` sends agree with the first with that to the power +/// `t − 1`. The feature is `None` unless the product over the repeated +/// needles is below `2^-40`. +pub fn repeated_needles_sent_alike(world: &ObservedWorld) -> Option { + fn needle(query: &ObservedQuery) -> Option<(&ObservedGroup, Option<&ObservedGroup>)> { + match query { + ObservedQuery::Contains { needle, alt } => Some((needle, alt.as_ref())), + ObservedQuery::Equals { needle } => Some((needle, None)), + _ => None, + } + } + fn ln_arrangements(items: &[T]) -> f64 { + let ln_factorial = |n: usize| -> f64 { (2..=n).map(|k| (k as f64).ln()).sum() }; + let mut sorted: Vec<&T> = items.iter().collect(); + sorted.sort_unstable(); + let alike: f64 = sorted + .chunk_by(|a, b| a == b) + .map(|run| ln_factorial(run.len())) + .sum(); + ln_factorial(items.len()) - alike + } + let mut repeats: BTreeMap> = BTreeMap::new(); + for (i, query) in world.queries.iter().enumerate() { + if let Some(content) = needle(query) { + repeats.entry(format!("{content:?}")).or_default().push(i); + } + } + let mut chance = 0.0f64; + let mut alike = true; + for sent in repeats.values().filter(|sent| sent.len() > 1) { + let (canonical, _) = needle(&world.queries[sent[0]]).expect("a needle"); + let elements: Vec = canonical + .elements + .iter() + .map(|g| format!("{g:?}")) + .collect(); + let once = ln_arrangements(&canonical.pairs) + ln_arrangements(&elements); + chance -= once * (sent.len() - 1) as f64; + let first = needle(&world.sent[sent[0]]); + alike &= sent.iter().all(|&i| needle(&world.sent[i]) == first); + } + (chance < -40.0 * 2f64.ln()).then_some(alike) +} + /// For every extraction, the positions in each row whose selector it sent: /// which rows `->` answers, and with how many entries. pub fn extraction_matches(world: &ObservedWorld) -> Vec>> { diff --git a/packages/stack-encrypt/tests/json_leakage/mod.rs b/packages/stack-encrypt/tests/json_leakage/mod.rs index 17513c5e3..06cf3423d 100644 --- a/packages/stack-encrypt/tests/json_leakage/mod.rs +++ b/packages/stack-encrypt/tests/json_leakage/mod.rs @@ -19,9 +19,11 @@ //! //! Since then a document's entries are stored in a uniformly random order //! drawn when it is sealed, the root at slot `0`, and `q`'s lists are -//! shuffled too ([`profile::StoredOrder`]); and each entry's AAD binds its -//! children's AEAD tags (the cascade), which leaves everything stored as it -//! was and so changes no clause. +//! shuffled too ([`profile::StoredOrder`]); a containment or equality +//! needle's lists are shuffled as well, afresh for each query +//! ([`profile::SentOrder`]); and each entry's AAD binds its children's AEAD +//! tags (the cascade), which leaves everything stored as it was and so +//! changes no clause. //! //! # The pieces //! @@ -45,7 +47,8 @@ //! [`profile::Leakage`] and randomness, with no plaintext and no key: //! random bytes for every name, random ciphertexts of the stated length, //! each row's entries at a random permutation of their slots with the root -//! first and `q`'s lists shuffled, as the crate stores them, +//! first and `q`'s lists shuffled, as the crate stores them, needles +//! shuffled afresh for each query, as the client sends them, //! ordering terms with CLLW's layout under a key of its own, over codes //! that have the stated order, lengths and shared prefixes and nothing //! else of the values, and the chain programmed @@ -56,19 +59,22 @@ //! through the crate's public API, and serializes them. //! - [`observe`] parses either world's wire JSON into one //! [`observe::ObservedWorld`]. -//! - [`canonical`] puts a world's entries and `q`'s lists in an order an -//! observer computes without the stored order, keeping each entry's stored -//! slot beside it. The stored order is random on both sides, independently, -//! so a feature that named an entry by its slot would tell a faithful -//! simulation from the real scheme by chance alone. +//! - [`canonical`] puts a world's entries, `q`'s lists and the needles' +//! lists in an order an observer computes without the stored or sent +//! order, keeping each entry's stored slot and each query as sent beside +//! them. Those orders are random on both sides, independently, so a +//! feature that named an entry by its slot, or read a needle's pairs as +//! sent, would tell a faithful simulation from the real scheme by chance +//! alone. //! - [`distinguishers`] are pure functions of an observed world, read in the //! canonical order: what an observer holding the column and the query //! terms could compute. Each feature is exact (byte strings renamed to //! integers in order of first appearance), so a real and a simulated world -//! are compared for equality, with no statistics. Two features need a +//! are compared for equality, with no statistics. Three features need a //! probability bound and carry it in their doc comments -//! ([`distinguishers::walked_subtrees_are_contiguous`], the one that reads -//! the stored order, and +//! ([`distinguishers::walked_subtrees_are_contiguous`], which reads the +//! stored order, [`distinguishers::repeated_needles_sent_alike`], which +//! reads the sent order, and //! [`distinguishers::member_order_shared_across_parents`]). //! - [`mutants`] are profiles with one clause dropped or coarsened. //! - [`corpus`] holds the handcrafted columns and a proptest generator of @@ -92,11 +98,12 @@ //! [`profile::Clauses::as_published_in_1363`] turns it off, and a test //! holds that the result is then told from the real scheme, by the //! distinguisher that names it and the three its absence breaks. -//! - **Three readings.** The key order (UTF-8 byte order, which a needle's -//! pairs are sent in), that strings order folded, and that a number's -//! equality under its term is as a 64-bit float; a profile read the other -//! way on any of them is told from the real scheme, so each is -//! load-bearing. +//! - **Readings.** That strings order folded, and that a number's equality +//! under its term is as a 64-bit float: a profile read the other way on +//! either is told from the real scheme, so each is load-bearing. The key +//! order is the converse: read as UTF-8 bytes or as `jsonb`'s sort, the +//! profiles differ and neither is told from the real scheme, since +//! nothing the database sees is in key order any more. //! - **The calibration.** Each mutant profile drops one clause. The test //! expects some distinguisher to tell each mutant's world from the real //! one, and prints a table of which (run with `--no-capture`). A mutant @@ -180,18 +187,15 @@ //! order before it (`stored_order_depth_first`) is told apart by //! `walked_subtrees_are_contiguous`. //! -//! Against the page as the random order leaves it: -//! -//! 3. **A needle still shows the key order.** A containment or equality -//! needle is sent in the walk's order, unshuffled, an object's members -//! in their keys' byte order, so a needle with two sibling keys tells the -//! database which sorts first, at paths whose selectors it also sees in -//! the stored rows. The stored `q` is shuffled; the needle is not. The -//! profile states it ([`profile::GroupLeakage::pairs`], the key-order -//! reading test, the mutant `needle_pairs_shuffled`, caught by -//! `transcripts`), and so does the page, under containment. Shuffling a -//! needle on the client, which the server reads as a set, would close -//! it; that is left to a decision of its own. +//! The random order left one place the key order still showed: a +//! containment or equality needle was sent in the walk's order, an object's +//! members in their keys' byte order, so a needle with two sibling keys told +//! the database which sorts first. The client now shuffles a needle as a +//! stored `q` is shuffled, afresh for each query, which the server reads as +//! a set anyway; the clause says so ([`profile::SentOrder`]), and the +//! mutant that states the walk's order before it +//! (`needles_sent_in_walk_order`) is told apart by +//! `repeated_needles_sent_alike`. //! //! # When the format changes //! @@ -304,21 +308,17 @@ //! //! Every simulation and every mutant draws from one seeded generator, and //! the random columns come from a seeded proptest runner. The real side -//! draws data keys, nonces and every document's stored order from the +//! draws data keys, nonces, every document's stored order and every +//! needle's sent order from the //! operating system. The keys and nonces reach a feature only through //! equality patterns, which change only on a collision of random bytes, //! below `n² / 2^64` for `n` eight-byte tags in a column (about `10^-13` -//! here). The stored order reaches none but through the canonical order, -//! which does not depend on it (ties it leaves are between entries no -//! feature tells apart), and through `walked_subtrees_are_contiguous`, -//! whose answer is fixed but with probability below `2^-40`. The fake key +//! here). The stored and sent orders reach none but through the canonical +//! order, which does not depend on them (ties it leaves are between names +//! and entries no feature tells apart), and through +//! `walked_subtrees_are_contiguous` and `repeated_needles_sent_alike`, whose +//! answers are fixed but with probability below `2^-40`. The fake key //! source's index key is fixed, so the real member ranks are too. -//! -//! The profile sorts members itself rather than trusting `serde_json`'s -//! iteration order, so a build that unified serde_json's `preserve_order` -//! feature on (which would send a needle in the application's field order) -//! fails the positive control on the handcrafted needle whose keys are -//! written out of order. pub mod calibration; pub mod canonical; diff --git a/packages/stack-encrypt/tests/json_leakage/mutants.rs b/packages/stack-encrypt/tests/json_leakage/mutants.rs index dfe01fc2a..22cccfbd9 100644 --- a/packages/stack-encrypt/tests/json_leakage/mutants.rs +++ b/packages/stack-encrypt/tests/json_leakage/mutants.rs @@ -11,7 +11,9 @@ //! code does not have, to show the distinguishers would see it: the //! pre-#1363 read's top-level token (`ancestor_token_sent`), the pre-#1363 //! member rule (`member_tokens_carry_ordinal`), the depth-first stored -//! order before the random one (`stored_order_depth_first`). +//! order before the random one (`stored_order_depth_first`), the walk's +//! order a needle was sent in before it was shuffled +//! (`needles_sent_in_walk_order`). use std::collections::{BTreeMap, HashMap}; @@ -20,7 +22,7 @@ use proptest::test_runner::TestRng; use super::profile::{ EntryLeakage, GroupLeakage, KindOrder, Leakage, OrderKind, Placement, QueryLeakage, ReadLeak, - RowLeakage, Rule, Step, StoredOrder, TokenName, TokenStep, AEAD_OVERHEAD, + RowLeakage, Rule, SentOrder, Step, StoredOrder, TokenName, TokenStep, AEAD_OVERHEAD, }; /// One mutant: its name, the clause it drops (a field or sentence of the @@ -775,16 +777,12 @@ fn needle_flat(l: &mut Leakage, _: &mut TestRng) { } } -/// A needle's pairs sent in a random order, as a stored `q`'s are stored: -/// what the profile would say if a needle's order revealed nothing. The -/// client sends a needle in the walk's order, an object's members in key -/// byte order, so a needle with two sibling keys tells their byte order. -fn needle_pairs_shuffled(l: &mut Leakage, rng: &mut TestRng) { - for query in &mut l.queries { - if let QueryLeakage::Contains { needle, .. } | QueryLeakage::Equals { needle } = query { - groups_mut(needle, &mut |g| shuffle(&mut g.pairs, rng)); - } - } +/// Every needle sent in the walk's order, an object's members in key byte +/// order: how the client sent a needle before it shuffled it, which told +/// the database which of two sibling keys sorts first, and sent one needle +/// in one order every time. +fn needles_sent_in_walk_order(l: &mut Leakage, _: &mut TestRng) { + l.sent_order = SentOrder::Walk; } fn needle_alt_dropped(l: &mut Leakage, _: &mut TestRng) { @@ -876,7 +874,7 @@ mutants!( token_list_padded => "QueryLeakage::Subtree::reads (the tokens' number, which gives N away)", budget_not_sent => "QueryLeakage::Subtree::budget (the budget, in the clear)", needle_flat => "QueryLeakage::Contains::needle (\"its element grouping\")", - needle_pairs_shuffled => "GroupLeakage::pairs of a needle (sent in the walk's order, members in key byte order)", + needles_sent_in_walk_order => "Leakage::sent_order (\"A needle is sent in a random order too, drawn afresh for each query\"): the walk's order before", needle_alt_dropped => "QueryLeakage::Contains::alt", bound_rank_random => "QueryLeakage::Order::bound (\"the bound's place in the order of the stored values\")", ); diff --git a/packages/stack-encrypt/tests/json_leakage/observe.rs b/packages/stack-encrypt/tests/json_leakage/observe.rs index 46091d4db..edf2b9408 100644 --- a/packages/stack-encrypt/tests/json_leakage/observe.rs +++ b/packages/stack-encrypt/tests/json_leakage/observe.rs @@ -10,9 +10,10 @@ //! lacks, or the other way round, shows in [`ObservedRow::fields`] and //! [`ObservedEntry::fields`]. //! -//! A world is parsed in its stored order. The distinguishers read it in the -//! canonical order ([`super::canonical`]), which keeps each entry's stored -//! slot beside it ([`ObservedRow::slots`]). +//! A world is parsed in its stored and sent order. The distinguishers read +//! it in the canonical order ([`super::canonical`]), which keeps each +//! entry's stored slot beside it ([`ObservedRow::slots`]) and each query as +//! sent ([`ObservedWorld::sent`]). use base64ct::{Base64, Encoding}; use serde_json::Value; @@ -120,6 +121,11 @@ pub enum ObservedQuery { pub struct ObservedWorld { pub rows: Vec, pub queries: Vec, + /// The query terms with their lists in the order they were sent: the + /// same as `queries` as parsed, and kept as they were after + /// [`super::canonical::canonical`] puts the needles in `queries` in the + /// canonical order. + pub sent: Vec, /// Every key path inside each query term's wire form ([`key_paths`]), /// which the parser above reads only in part. pub query_fields: Vec>, @@ -129,9 +135,11 @@ impl ObservedWorld { /// Parse a world from its wire form: each row as its stored JSON, each /// query as the JSON its term serializes to. pub fn from_wire(rows: &[Value], queries: &[(QueryKind, Value)]) -> Self { + let parsed: Vec = queries.iter().map(|(kind, q)| query(*kind, q)).collect(); Self { rows: rows.iter().map(row).collect(), - queries: queries.iter().map(|(kind, q)| query(*kind, q)).collect(), + sent: parsed.clone(), + queries: parsed, query_fields: queries.iter().map(|(_, q)| key_paths(q)).collect(), } } diff --git a/packages/stack-encrypt/tests/json_leakage/profile.rs b/packages/stack-encrypt/tests/json_leakage/profile.rs index aa849acc3..f09be126b 100644 --- a/packages/stack-encrypt/tests/json_leakage/profile.rs +++ b/packages/stack-encrypt/tests/json_leakage/profile.rs @@ -31,10 +31,12 @@ //! //! Since then a document's entries are stored in a uniformly random order //! drawn when it is sealed, the root first, and `q`'s lists are shuffled -//! the same way ([`Leakage::stored_order`]); each entry's AAD binds its -//! children's AEAD tags (the cascade), which changes nothing stored. The -//! profile numbers a row's nodes depth-first ([`flatten`]) for its own -//! bookkeeping: that numbering is not the stored order. +//! the same way ([`Leakage::stored_order`]); a needle's lists are sent +//! shuffled too, afresh for each query ([`Leakage::sent_order`]); each +//! entry's AAD binds its children's AEAD tags (the cascade), which changes +//! nothing stored. The profile numbers a row's nodes, and a needle's, +//! depth-first ([`flatten`]) for its own bookkeeping: that numbering is not +//! the order either is stored or sent in. use std::collections::{BTreeMap, HashMap}; use std::hash::Hash; @@ -111,12 +113,14 @@ impl Query { } } -/// Which sort "the byte order of their key names' UTF-8 encodings" means. -/// The stored order no longer shows it ([`StoredOrder::Random`]), but a -/// containment or equality needle is still sent in the walk's order, an -/// object's members in this order ([`GroupLeakage::pairs`]), so a needle -/// with two sibling keys tells the database their byte order. The other -/// reading is kept to show that is load-bearing. +/// Which sort "the byte order of their key names' UTF-8 encodings" means: +/// the order of an object's members in the walk's order, which numbers the +/// profile's nodes ([`flatten`]). Nothing the database sees shows it now: +/// the stored order is random ([`StoredOrder::Random`]), a needle is sent in +/// a random order ([`SentOrder::Random`]), and a member's ordinal is a keyed +/// rank. The other reading is kept to show that: a profile read either way +/// is told from the real scheme by no distinguisher (the key-order reading +/// test in `calibration.rs`). #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum KeyOrder { /// Byte order of the UTF-8 key: serde_json's `Map` without @@ -156,8 +160,7 @@ pub enum NumberOrder { /// readings the page could leave open. The default is what the code needs. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct Clauses { - /// How an object's members sort in the walk's order, which is the order - /// a needle's pairs are sent in. + /// How an object's members sort in the walk's order. pub key_order: KeyOrder, /// How a string's ordering term orders. pub string_order: StringOrder, @@ -345,8 +348,9 @@ pub struct EntryLeakage { pub struct GroupLeakage { /// "One `{s, vs}` pair per node": the pairs of the nodes that sit /// directly in this group, in the walk's order, an object's members in - /// [`Clauses::key_order`]. A needle is sent in this order; a stored - /// `q`'s lists are stored shuffled ([`Leakage::stored_order`]). "Whether + /// [`Clauses::key_order`]. Neither list is stored or sent in this order: + /// a stored `q`'s lists are shuffled ([`Leakage::stored_order`]), and so + /// are a needle's ([`Leakage::sent_order`]). "Whether /// two rows hold the same value at the same path, through the value /// selector `vs`, a keyed hash of the path and the value (for an object /// or an array, of its kind alone)." @@ -489,6 +493,23 @@ pub enum StoredOrder { DepthFirst, } +/// The order a containment or equality needle's lists are sent in. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum SentOrder { + /// The page: "A needle is sent in a random order too, drawn afresh for + /// each query, so it shows no more of the order of the needle's keys + /// than a stored row does." Every pair list and every element-group list + /// of a needle (and its one-pair `alt`) is a uniformly random + /// permutation, drawn independently for each query, so two queries for + /// one needle send its pairs in two unrelated orders. + Random, + /// The walk's order, an object's members in [`Clauses::key_order`]: the + /// order a needle was sent in before the client shuffled it, which told + /// the database which of two sibling keys sorts first. Only a mutant + /// states it now. + Walk, +} + /// The leakage of a whole column: its rows and its queries, over one shared /// set of opaque names. #[derive(Clone, Debug, PartialEq, Eq)] @@ -498,6 +519,8 @@ pub struct Leakage { pub key_tag_len: usize, /// The order every row's entries and `q` are stored in. pub stored_order: StoredOrder, + /// The order every needle is sent in. + pub sent_order: SentOrder, pub rows: Vec, pub queries: Vec, /// The rank of each [`OrderClass`] among the classes of its kind: "the @@ -798,8 +821,8 @@ pub struct Node<'a> { /// A document's nodes in the walk's order: depth-first, the root first, an /// object's members in `order`, an array's elements in position order. The -/// profile's numbering of a row's nodes, and the order of a needle's pairs; -/// not the stored order ([`StoredOrder`]). +/// profile's numbering of a row's nodes and of a needle's; not the order +/// either is stored or sent in ([`StoredOrder`], [`SentOrder`]). pub fn flatten(doc: &Value, order: KeyOrder) -> Vec> { let mut flattener = Flattener { order, @@ -1241,6 +1264,7 @@ pub fn leak(column: &Column, clauses: Clauses, docs: &[Value], queries: &[Query] Leakage { key_tag_len: KEY_TAG_LEN, stored_order: StoredOrder::Random, + sent_order: SentOrder::Random, rows, queries, ordering_ranks: ids.ordering_ranks(), diff --git a/packages/stack-encrypt/tests/json_leakage/simulator.rs b/packages/stack-encrypt/tests/json_leakage/simulator.rs index 713c8b9ba..efef87b74 100644 --- a/packages/stack-encrypt/tests/json_leakage/simulator.rs +++ b/packages/stack-encrypt/tests/json_leakage/simulator.rs @@ -32,8 +32,9 @@ //! nodes at a uniformly random permutation of the slots `1..n`, and every //! list of the stored `q` shuffled, each by Fisher–Yates over the //! simulator's generator. The profile names nodes, not slots, so the tags -//! the chain programs follow each node to wherever it is stored. A needle is -//! sent in the walk's order, unshuffled, as the client sends it. +//! the chain programs follow each node to wherever it is stored. A needle's +//! lists are shuffled the same way, afresh for each query +//! ([`SentOrder::Random`]), as the client shuffles them. use std::collections::{BTreeMap, HashMap}; @@ -49,7 +50,7 @@ use super::chain::{ use super::observe::QueryKind; use super::profile::{ GroupLeakage, KindOrder, Leakage, OrderKind, OrderingLeak, Placement, QueryLeakage, RowLeakage, - Step, StoredOrder, TokenName, TokenStep, + SentOrder, Step, StoredOrder, TokenName, TokenStep, }; /// The version byte `vitaminc-encrypt` writes at the front of every @@ -332,9 +333,9 @@ impl Simulator<'_> { bytes.clone() } - /// A projection's wire form: in the walk's order, as a needle is sent, - /// or with every list shuffled, as a stored `q` is under - /// [`StoredOrder::Random`]. + /// A projection's wire form: in the walk's order, or with every list + /// shuffled, as a stored `q` is under [`StoredOrder::Random`] and a + /// needle under [`SentOrder::Random`]. fn group(&mut self, group: &GroupLeakage, shuffled: bool) -> Value { let mut pairs: Vec = group .pairs @@ -550,13 +551,17 @@ impl Simulator<'_> { (QueryKind::Subtree, json!({"t": tokens, "budget": budget})) } QueryLeakage::Contains { needle, alt } => { - let mut wire = self.group(needle, false); + let shuffled = self.leakage.sent_order == SentOrder::Random; + let mut wire = self.group(needle, shuffled); if let Some(alt) = alt { - wire["alt"] = self.group(alt, false); + wire["alt"] = self.group(alt, shuffled); } (QueryKind::Contains, wire) } - QueryLeakage::Equals { needle } => (QueryKind::Equals, self.group(needle, false)), + QueryLeakage::Equals { needle } => { + let shuffled = self.leakage.sent_order == SentOrder::Random; + (QueryKind::Equals, self.group(needle, shuffled)) + } QueryLeakage::Order { selector, bound } => { let s = self.selector(*selector); let op = self.ordering(bound.class, bound.len); From 3ed9d3b7607bdd41db9eb9b33c8ad23333c0ed4b Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 16:17:43 +1100 Subject: [PATCH 10/22] fix(stack-encrypt): wipe the JSON index's key names, values and data keys before freeing them The data-plan work in #1372 found that sealing a JSON document freed key names, scalar values and the data key unwiped, and left it for the index work. A heap spy over this branch's seal, open, read, extraction and needle paths, under the deterministic key source, found 42 blocks for one sealed-root seal (43 with the root open), 4 for each whole open and each `->`, 7 for each subtree read, 40 for a containment needle and 20 for an equality needle. Each holds a marker the document carries or the data key the seal used. After this change every one of those paths frees none. Where they came from, and what holds them now: - JsonPath carries key names, and the walk builds one per node: a path now wipes its key names when it drops, and `selector_bytes` and `token_bytes` are sized before they are written (a growing buffer freed a copy) and come back in `Zeroizing`. - The value selector framed the scalar, and the rank framed the key name, through `Context::pae`, whose buffer is not wiped: both now frame into a sized `Zeroizing` buffer with the same bytes (a unit test holds the equality), and the canonical value is `Zeroizing` too. The rank map no longer clones key names. - The document `seal_pending` owns (a clone, in borrowed mode) is now wiped when sealing returns (`wipe_value`); a query's needle, owned by the terms operation, is wiped through a new `SourceMode::release`; the copies `AtPath` checks make are wiped after the check. - The data key was moved out of the response buffers (`pop_front`, `drain`, `into_iter`), leaving its bytes in memory freed later. Each response is now handed out as a clone and the original drops in place, where `DataKey` wipes itself; `dispatch` clones retrieved keys the same way. This is the shared settling path, so every decryption benefits. `tests/heap_hygiene.rs` gains the JSON index's tests, with a key source that records each data key so the spy looks for it as well as the marker, and a control showing a freed key copy fails the watch. Each fix was checked by removing it and watching a test fail. One case is left out and written down: the path an `AtPath` caller passes as an owned `String` is cloned by the query builder and dropped by the plan chain, which knows only `P: AsRef`. Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- packages/stack-encrypt/src/json/derive.rs | 120 +++++-- packages/stack-encrypt/src/json/mod.rs | 2 +- packages/stack-encrypt/src/json/path.rs | 65 +++- packages/stack-encrypt/src/json/query.rs | 19 +- .../stack-encrypt/src/target/operations.rs | 12 +- packages/stack-encrypt/src/target/pending.rs | 12 +- packages/stack-encrypt/src/target/request.rs | 68 +++- packages/stack-encrypt/src/target/source.rs | 8 + packages/stack-encrypt/tests/heap_hygiene.rs | 314 +++++++++++++++++- 9 files changed, 567 insertions(+), 53 deletions(-) diff --git a/packages/stack-encrypt/src/json/derive.rs b/packages/stack-encrypt/src/json/derive.rs index 0838decef..22f3f1435 100644 --- a/packages/stack-encrypt/src/json/derive.rs +++ b/packages/stack-encrypt/src/json/derive.rs @@ -20,6 +20,7 @@ use vitaminc_encrypt::{Aes256Cipher, Key as AesKey}; use vitaminc_hmac::HmacSha256Prf; use vitaminc_prf::{BlockVisitor, IntoPrfContext, PrfValue}; use vitaminc_protected::{AsProtectedRef, Controlled, Protected}; +use zeroize::{Zeroize, Zeroizing}; use super::cascade::{self, ChildrenDigest}; use super::order; @@ -463,17 +464,30 @@ fn rank( parent: &Tag, key: &str, ) -> Result { - let framed = Context::pae(&[parent, key.as_bytes()]); - prf16(prf, RANK_DOMAIN, context, framed.as_bytes()) + let framed = pae_wiped(&[parent, key.as_bytes()]); + prf16(prf, RANK_DOMAIN, context, &framed) +} + +/// [`Context::pae`] into a buffer that is sized before it is written and +/// wiped when it drops: for framing that holds a key name or a scalar. +fn pae_wiped(pieces: &[&[u8]]) -> Zeroizing> { + let len = 8 + pieces.iter().map(|piece| 8 + piece.len()).sum::(); + let mut out = Zeroizing::new(Vec::with_capacity(len)); + out.extend_from_slice(&(pieces.len() as u64).to_le_bytes()); + for piece in pieces { + out.extend_from_slice(&(piece.len() as u64).to_le_bytes()); + out.extend_from_slice(piece); + } + out } /// A number's canonical bytes for its value selector: an integral value, /// whichever JSON spelling it had, is its `i128`; anything else is its /// `f64`. So `2` and `2.0` are one value, as `jsonb` compares them, and /// `-0.0` is `0`. -fn canonical_number(number: &Number) -> Vec { - fn int(i: i128) -> Vec { - let mut out = Vec::with_capacity(17); +fn canonical_number(number: &Number) -> Zeroizing> { + fn int(i: i128) -> Zeroizing> { + let mut out = Zeroizing::new(Vec::with_capacity(17)); out.push(b'i'); out.extend_from_slice(&i.to_le_bytes()); out @@ -488,7 +502,7 @@ fn canonical_number(number: &Number) -> Vec { if f.fract() == 0.0 && f.abs() < 1e38 { return int(f as i128); } - let mut out = Vec::with_capacity(9); + let mut out = Zeroizing::new(Vec::with_capacity(9)); out.push(b'f'); out.extend_from_slice(&f.to_le_bytes()); out @@ -496,12 +510,12 @@ fn canonical_number(number: &Number) -> Vec { /// A scalar's canonical bytes; a container contributes nothing beyond its /// kind's tag. -fn canonical(value: &Value) -> Vec { +fn canonical(value: &Value) -> Zeroizing> { match value { - Value::Null | Value::Object(_) | Value::Array(_) => Vec::new(), - Value::Bool(b) => vec![u8::from(*b)], + Value::Null | Value::Object(_) | Value::Array(_) => Zeroizing::new(Vec::new()), + Value::Bool(b) => Zeroizing::new(vec![u8::from(*b)]), Value::Number(n) => canonical_number(n), - Value::String(s) => s.as_bytes().to_vec(), + Value::String(s) => Zeroizing::new(s.as_bytes().to_vec()), } } @@ -513,8 +527,8 @@ fn value_selector( s: &Tag, value: &Value, ) -> Result { - let framed = Context::pae(&[Kind::of(value).tag(), s, &canonical(value)]); - prf16(prf, VALUE_DOMAIN, context, framed.as_bytes()) + let framed = pae_wiped(&[Kind::of(value).tag(), s, &canonical(value)]); + prf16(prf, VALUE_DOMAIN, context, &framed) } /// The leaf ordering term: the OPE term of a number as an `f64` or of a @@ -684,7 +698,7 @@ impl Walker<'_> { }; match value { Value::Object(map) => { - let ordinals: Option> = chains + let ordinals: Option> = chains .then(|| shuffled_ordinals(self.prf, self.context, &token, map)) .transpose()?; for (key, child) in map { @@ -732,12 +746,12 @@ impl Walker<'_> { /// An object's members' ordinals: their positions in the order of their /// ranks under the parent's token, so the ordinals rank nothing a reader /// without the token can see. -fn shuffled_ordinals( +fn shuffled_ordinals<'m>( prf: &HmacSha256Prf, context: &Context<'_>, parent: &Tag, - map: &Map, -) -> Result, TermError> { + map: &'m Map, +) -> Result, TermError> { let mut ranked: Vec<(Tag, &String)> = map .keys() .map(|k| Ok((rank(prf, context, parent, k)?, k))) @@ -746,7 +760,7 @@ fn shuffled_ordinals( Ok(ranked .into_iter() .enumerate() - .map(|(o, (_, key))| (key.clone(), o as u32)) + .map(|(o, (_, key))| (key, o as u32)) .collect()) } @@ -793,6 +807,8 @@ pub(crate) fn seal_pending<'a, K: 'static>( if let Err(error) = descriptor.check() { return Pending::failed(cipher, error); } + // The caller's document, owned here: wiped when this returns. + let value = Wiped(value); let prf_context = piece.clone().into_prf_context().into_owned(); let aad = piece.into_aad().into_owned(); let walked = match walk(&value, options, cipher.prf(), &prf_context) { @@ -823,6 +839,39 @@ pub(crate) fn seal_pending<'a, K: 'static>( ) } +/// A document this crate owns, its strings and key names wiped when it +/// drops. +pub(crate) struct Wiped(pub(crate) Value); + +impl std::ops::Deref for Wiped { + type Target = Value; + fn deref(&self) -> &Value { + &self.0 + } +} + +impl Drop for Wiped { + fn drop(&mut self) { + wipe_value(&mut self.0); + } +} + +/// Wipe every string and key name in `value`, leaving it empty: numbers, +/// booleans and nulls hold nothing outside the value itself. +pub(crate) fn wipe_value(value: &mut Value) { + match value { + Value::String(text) => text.zeroize(), + Value::Array(items) => items.iter_mut().for_each(wipe_value), + Value::Object(map) => { + for (mut key, mut child) in std::mem::take(map) { + wipe_value(&mut child); + key.zeroize(); + } + } + Value::Null | Value::Bool(_) | Value::Number(_) => {} + } +} + /// An entry's AAD: the domain, the keyset, the column's AAD, the key tag, /// the entry's selector, the slot `j` it is stored at, and the digest of its /// children's AEAD tags ([`cascade::children_digest`]). The last is what @@ -1697,7 +1746,7 @@ impl<'k> Placing<'k> { if !names(path, &label) { return Err(refused()); } - Protected::new(path.selector_bytes()) + path.protected_selector_bytes() } Under::Node(parent) => child_selector_bytes(parent, &label).ok_or_else(refused)?, }; @@ -2172,6 +2221,35 @@ mod tests { .terms } + /// The wiped framing is byte for byte the PAE vitaminc writes, so a + /// value selector or a rank does not move. + #[test] + fn the_wiped_framing_is_vitamincs_pae() { + for pieces in [ + vec![], + vec![&b""[..]], + vec![&b"a"[..], &b""[..], &b"key name"[..]], + vec![&[7u8; 16][..], &b"x".repeat(300)[..]], + ] { + let wiped = pae_wiped(&pieces); + assert_eq!(&wiped[..], Context::pae(&pieces).as_bytes()); + assert_eq!(wiped.len(), wiped.capacity(), "sized before it is written"); + } + } + + #[test] + fn wiping_a_value_empties_its_strings_and_keys() { + let mut value = json!({"key": ["text", 1, null, {"inner": "more"}], "n": 2.5}); + wipe_value(&mut value); + assert_eq!(value, json!({}), "every member taken and wiped"); + let mut text = json!("text"); + wipe_value(&mut text); + assert_eq!(text, json!("")); + let mut list = json!(["a", ["b"], true]); + wipe_value(&mut list); + assert_eq!(list, json!(["", [""], true])); + } + #[test] fn numbers_canonicalise_by_value() { let n = |text: &str| -> Number { text.parse().unwrap() }; @@ -2883,12 +2961,12 @@ mod tests { #[test] fn a_childs_selector_bytes_are_its_paths() { - let parent = Protected::new(path("$.a[3]").selector_bytes()); + let parent = path("$.a[3]").protected_selector_bytes(); for (label, child) in [(key("b"), "$.a[*].b"), (Label::Index(7), "$.a[*][*]")] { let bytes = child_selector_bytes(&parent, &label).expect("a child"); assert_eq!( bytes.risky_ref(), - &path(child).selector_bytes(), + &*path(child).selector_bytes(), "{child}: the parent's bytes and the child's step" ); assert_eq!( @@ -2953,7 +3031,7 @@ mod tests { }; assert_eq!( selectors - .of(Protected::new(path("$.a.b").selector_bytes())) + .of(path("$.a.b").protected_selector_bytes()) .unwrap(), at("$.a.b"), "a protected input derives what its bytes do" diff --git a/packages/stack-encrypt/src/json/mod.rs b/packages/stack-encrypt/src/json/mod.rs index 95e4b5fec..3ea0f880e 100644 --- a/packages/stack-encrypt/src/json/mod.rs +++ b/packages/stack-encrypt/src/json/mod.rs @@ -135,7 +135,7 @@ pub use query::{ AtPathTerms, JsonLeafQuery, JsonProjection, JsonQuery, JsonSelector, JsonSubtreeKey, }; -pub(crate) use derive::{seal_pending, terms_pending}; +pub(crate) use derive::{seal_pending, terms_pending, wipe_value}; /// A sixteen-byte tag: a selector or a token. pub type Tag = [u8; 16]; diff --git a/packages/stack-encrypt/src/json/path.rs b/packages/stack-encrypt/src/json/path.rs index 77f86a457..e1eb8fa31 100644 --- a/packages/stack-encrypt/src/json/path.rs +++ b/packages/stack-encrypt/src/json/path.rs @@ -16,6 +16,8 @@ use std::fmt; use std::str::FromStr; +use zeroize::{Zeroize, Zeroizing}; + /// One step of a [`JsonPath`]. #[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] pub enum Segment { @@ -28,11 +30,25 @@ pub enum Segment { } /// A path into a JSON document, from the root (`$`) down. +/// +/// A key step's name can be customer data (the walk builds a path for every +/// node of a document it seals or queries), so a path wipes its key names +/// when it drops. #[derive(Clone, Debug, PartialEq, Eq, Hash, Default, PartialOrd, Ord)] pub struct JsonPath { segments: Vec, } +impl Drop for JsonPath { + fn drop(&mut self) { + for segment in &mut self.segments { + if let Segment::Key(key) = segment { + key.zeroize(); + } + } + } +} + /// Why text is not a [`JsonPath`]. The message names the position of the /// first character the parser refused and nothing else of the text: a path /// can name a key that is customer data. @@ -260,8 +276,18 @@ impl JsonPath { /// positional or wildcard, is the one array tag, so every element of an /// array shares its parent's selector. A key is length-prefixed, so two /// paths cannot encode to one byte string. - pub(crate) fn selector_bytes(&self) -> Vec { - let mut out = Vec::new(); + pub(crate) fn selector_bytes(&self) -> Zeroizing> { + // Sized before it is written: a buffer that grew would free a copy + // of the key names it held. + let len = self + .segments + .iter() + .map(|segment| match segment { + Segment::Key(key) => key_step_len(key.len()), + Segment::Index(_) | Segment::Wildcard => 1, + }) + .sum(); + let mut out = Zeroizing::new(Vec::with_capacity(len)); for segment in &self.segments { match segment { Segment::Key(key) => push_key(&mut out, key), @@ -271,11 +297,24 @@ impl JsonPath { out } + /// [`selector_bytes`](Self::selector_bytes), moved into a `Protected`. + pub(crate) fn protected_selector_bytes(&self) -> vitaminc_protected::Protected> { + vitaminc_protected::Protected::new(std::mem::take(&mut *self.selector_bytes())) + } + /// The canonical bytes a token is derived over: an array step carries /// its position. `None` for a path with a wildcard, which names no one /// node. - pub(crate) fn token_bytes(&self) -> Option> { - let mut out = Vec::new(); + pub(crate) fn token_bytes(&self) -> Option>> { + let len = self + .segments + .iter() + .map(|segment| match segment { + Segment::Key(key) => key_step_len(key.len()), + Segment::Index(_) | Segment::Wildcard => 1 + 8, + }) + .sum(); + let mut out = Zeroizing::new(Vec::with_capacity(len)); for segment in &self.segments { match segment { Segment::Key(key) => push_key(&mut out, key), @@ -540,6 +579,24 @@ mod tests { /// `covers` is public and matches positions though an ordered path may /// not name one: a position covers that position alone. + /// The byte encodings hold key names, so each is sized before it is + /// written: a buffer that grew would free a copy. (That they and a path + /// wipe on drop is held by `tests/heap_hygiene.rs`.) + #[test] + fn the_byte_encodings_are_sized_once() { + use vitaminc_protected::Controlled; + for text in ["$", "$.a", "$.a[3].bc", r#"$["a b"][*][0]"#, "$[1][2].k"] { + let path = JsonPath::parse(text).unwrap(); + let selector = path.selector_bytes(); + assert_eq!(selector.len(), selector.capacity(), "{text}: selector"); + if let Some(token) = path.token_bytes() { + assert_eq!(token.len(), token.capacity(), "{text}: token"); + } + let protected = path.protected_selector_bytes(); + assert_eq!(protected.risky_ref(), &*selector, "{text}: protected"); + } + } + #[test] fn a_position_covers_its_own_position_only() { let path = |text: &str| JsonPath::parse(text).unwrap(); diff --git a/packages/stack-encrypt/src/json/query.rs b/packages/stack-encrypt/src/json/query.rs index 11796199b..755ef8381 100644 --- a/packages/stack-encrypt/src/json/query.rs +++ b/packages/stack-encrypt/src/json/query.rs @@ -718,7 +718,7 @@ where type Verifier = (); fn check(&self, (path, value): &(P, V)) -> Result<(), Error> { let _ = parse_path(path.as_ref())?; - scalar(&value.clone().into()) + wiped_after(value.clone().into(), scalar) } fn verifier(&self, _: &(P, V)) {} fn query((): (), terms: AtPathTerms, _: QueryToken) -> JsonQuery { @@ -741,6 +741,17 @@ where } /// Whether `value` is a scalar, which equality at a path compares. +/// `check` over `value`, a copy this crate made of the caller's plaintext, +/// which is wiped afterwards rather than dropped. +fn wiped_after( + mut value: Value, + check: impl FnOnce(&Value) -> Result<(), Error>, +) -> Result<(), Error> { + let checked = check(&value); + super::wipe_value(&mut value); + checked +} + fn scalar(value: &Value) -> Result<(), Error> { match value { Value::Object(_) | Value::Array(_) => Err(TermError::JsonValueNotScalar.into()), @@ -788,7 +799,7 @@ macro_rules! orderings { fn check(&self, (path, value): &(P, V)) -> Result<(), Error> { let path = parse_path(path.as_ref())?; orders(self, &path)?; - ordered(&value.clone().into()) + wiped_after(value.clone().into(), ordered) } fn verifier(&self, _: &(P, V)) {} fn query((): (), terms: AtPathTerms, _: QueryToken) -> JsonLeafQuery { @@ -823,8 +834,8 @@ where fn check(&self, (path, (lo, hi)): &(P, (V, V))) -> Result<(), Error> { let path = parse_path(path.as_ref())?; orders(self, &path)?; - ordered(&lo.clone().into())?; - ordered(&hi.clone().into()) + wiped_after(lo.clone().into(), ordered)?; + wiped_after(hi.clone().into(), ordered) } fn verifier(&self, _: &(P, (V, V))) {} fn query( diff --git a/packages/stack-encrypt/src/target/operations.rs b/packages/stack-encrypt/src/target/operations.rs index 09b0b2f76..b74fec3af 100644 --- a/packages/stack-encrypt/src/target/operations.rs +++ b/packages/stack-encrypt/src/target/operations.rs @@ -575,12 +575,16 @@ pub fn json_terms<'s, K: 'static, M: SourceMode<'s, serde_json::Value>>( options: JsonOptions, ) -> Encryption<'s, serde_json::Value, JsonTerms, K, CallerContext, M> { Encryption { - build: Box::new( - move |source, cipher, cx: CallerContext| match cx.validated() { + build: Box::new(move |source, cipher, cx: CallerContext| { + let pending = match cx.validated() { Ok(ctx) => crate::json::terms_pending(M::view(&source), &options, cipher, ctx), Err(e) => Pending::failed(cipher, e), - }, - ), + }; + // The terms are derived by now; a document this description + // owns (a query's needle) is wiped rather than dropped. + M::release(source, |mut value| crate::json::wipe_value(&mut value)); + pending + }), } } diff --git a/packages/stack-encrypt/src/target/pending.rs b/packages/stack-encrypt/src/target/pending.rs index 116c90029..9e3981ee5 100644 --- a/packages/stack-encrypt/src/target/pending.rs +++ b/packages/stack-encrypt/src/target/pending.rs @@ -567,15 +567,19 @@ async fn dispatch( received: keys.len(), }); } - for (index, key) in indices.into_iter().zip(keys) { - retrieved[index] = Some(key); + // Cloned, not moved: a data key wipes itself when it drops, and one + // moved out of `keys` would leave its bytes in the buffer `keys` + // frees. The originals drop here, in place. + for (index, key) in indices.into_iter().zip(&keys) { + retrieved[index] = Some(key.clone()); } } // Every slot was filled by exactly one group; a hole would mean the // grouping above lost a request, which is a bug here, not a data error. + // Cloned out for the same reason; `retrieved` drops its keys in place. let retrieved: Vec = retrieved - .into_iter() - .map(|key| key.ok_or(Error::ResponseShape)) + .iter() + .map(|key| key.clone().ok_or(Error::ResponseShape)) .collect::>()?; // A keyset request is answered from the cipher's cache, and loaded from diff --git a/packages/stack-encrypt/src/target/request.rs b/packages/stack-encrypt/src/target/request.rs index 3dadb7a33..e94e76c9b 100644 --- a/packages/stack-encrypt/src/target/request.rs +++ b/packages/stack-encrypt/src/target/request.rs @@ -12,7 +12,6 @@ //! sibling's key material are unit-testable on their own. use super::context::AeadContext; -use std::collections::VecDeque; use std::sync::Arc; use stack_kms::{DataKey, DataKeyWithTag, Iv}; @@ -157,25 +156,68 @@ pub(super) fn tally(requests: &[Request]) -> (usize, usize, usize) { /// asked for (never a neighbour's), and drawing past that is /// [`Error::ResponseShape`]. pub struct Responses { - generated: VecDeque, - retrieved: VecDeque, - keysets: VecDeque>, + generated: Queue, + retrieved: Queue, + keysets: Queue>, +} + +/// One kind's responses, drawn front first. A response is handed out as a +/// clone and the original stays where it is until the queue drops, so it +/// drops in place: a data key wipes itself on drop, and one moved out of the +/// buffer would leave its bytes behind in the freed allocation. +struct Queue { + items: Vec, + next: usize, +} + +impl Queue { + fn new(items: Vec) -> Self { + Self { items, next: 0 } + } + + fn len(&self) -> usize { + self.items.len() - self.next + } + + fn is_empty(&self) -> bool { + self.len() == 0 + } + + fn pop_front(&mut self) -> Option { + let item = self.items.get(self.next).cloned()?; + self.next += 1; + Some(item) + } + + /// The first `n`, as a queue of their own; `n` must not pass the end. + fn split_front(&mut self, n: usize) -> Self { + let front = self.items[self.next..self.next + n].to_vec(); + self.next += n; + Self::new(front) + } + + /// Every response left, front first. + fn drain(&mut self) -> impl Iterator + '_ { + let rest = &self.items[self.next..]; + self.next = self.items.len(); + rest.iter().cloned() + } } impl Responses { /// The full response set of one batched dispatch, in request order. pub(super) fn new(generated: Vec, retrieved: Vec) -> Self { Self { - generated: generated.into(), - retrieved: retrieved.into(), - keysets: VecDeque::new(), + generated: Queue::new(generated), + retrieved: Queue::new(retrieved), + keysets: Queue::new(Vec::new()), } } /// These responses with the keysets loaded for the keyset requests, in /// request order. pub(super) fn with_keysets(mut self, keysets: Vec>) -> Self { - self.keysets = keysets.into(); + self.keysets = Queue::new(keysets); self } @@ -211,9 +253,9 @@ impl Responses { return Err(Error::ResponseShape); } Ok(Responses { - generated: self.generated.drain(..generated).collect(), - retrieved: self.retrieved.drain(..retrieved).collect(), - keysets: self.keysets.drain(..keysets).collect(), + generated: self.generated.split_front(generated), + retrieved: self.retrieved.split_front(retrieved), + keysets: self.keysets.split_front(keysets), }) } @@ -225,11 +267,11 @@ impl Responses { } pub(crate) fn drain_generated(&mut self) -> impl Iterator + '_ { - self.generated.drain(..) + self.generated.drain() } pub(crate) fn drain_retrieved(&mut self) -> impl Iterator + '_ { - self.retrieved.drain(..) + self.retrieved.drain() } } diff --git a/packages/stack-encrypt/src/target/source.rs b/packages/stack-encrypt/src/target/source.rs index 09318e1ff..a3a342166 100644 --- a/packages/stack-encrypt/src/target/source.rs +++ b/packages/stack-encrypt/src/target/source.rs @@ -57,6 +57,10 @@ pub trait SourceMode<'s, S: 's>: sealed::Sealed + 's { /// Read the plaintext in place, for an operation that does not consume /// it (a match term reads text through `AsRef`). fn view(source: &Self::Source) -> &S; + /// Let go of the plaintext after reading it in place: in [`Owned`] + /// mode the description owns it, and hands it to `wipe`; in + /// [`Borrowed`] mode it is the caller's, and nothing is done. + fn release(source: Self::Source, wipe: impl FnOnce(S)); } /// A mode in which an operation can take the plaintext by value: always for @@ -79,6 +83,7 @@ impl<'s, S: 's> SourceMode<'s, S> for Borrowed { fn view(source: &Self::Source) -> &S { source } + fn release(_: Self::Source, _: impl FnOnce(S)) {} } impl<'s, S: Clone + 's> ConsumeSource<'s, S> for Borrowed { fn take(source: Self::Source) -> S { @@ -96,6 +101,9 @@ impl<'s, S: 's> SourceMode<'s, S> for Owned { fn view(source: &Self::Source) -> &S { source } + fn release(source: Self::Source, wipe: impl FnOnce(S)) { + wipe(source); + } } impl<'s, S: 's> ConsumeSource<'s, S> for Owned { fn take(source: Self::Source) -> S { diff --git a/packages/stack-encrypt/tests/heap_hygiene.rs b/packages/stack-encrypt/tests/heap_hygiene.rs index 0908d3b25..8903431b2 100644 --- a/packages/stack-encrypt/tests/heap_hygiene.rs +++ b/packages/stack-encrypt/tests/heap_hygiene.rs @@ -1,4 +1,13 @@ -//! No heap block the dynamic paths free still holds plaintext. +//! No heap block the dynamic paths and the JSON index free still holds +//! plaintext, or (for the JSON index) the data key a document is sealed +//! under. +//! +//! The JSON index's tests ([`json_index`]) cover sealing under either root +//! option, opening a document whole, a subtree read, an extraction and the +//! containment and equality needles. One thing they leave out: the path a +//! caller hands `AtPath` as an owned `String` is cloned by the query +//! builder and dropped by the plan chain unwiped, since the chain only knows +//! `P: AsRef`; the test passes a `&'static str`. //! //! This binary installs its own global allocator: a spy that, while a test //! watches its own thread, scans every block freed on that thread for a @@ -47,6 +56,11 @@ static GATE: Mutex<()> = Mutex::new(()); static FOUND: Mutex> = Mutex::new(Vec::new()); static UPSTREAM: Mutex = Mutex::new(0); +/// The data key the JSON tests' key source last handed out, which a freed +/// block must not hold either (see [`json_index`]). Set outside any +/// allocation: a fixed array behind a lock. +static KEY: Mutex> = Mutex::new(None); + /// The vitaminc frame whose free is excluded (see the module docs). const UPSTREAM_SEAL: &str = "vitaminc_encrypt::backend::aws_lc::CipherKey::seal"; @@ -70,6 +84,36 @@ unsafe fn holds_mark(ptr: *const u8, len: usize) -> bool { !memmem(ptr.cast(), len, MARK.as_ptr().cast(), MARK.len()).is_null() } +/// Whether the `len` bytes at `ptr` hold the data key in [`KEY`], read as +/// [`holds_mark`] reads. +/// +/// # Safety +/// +/// `ptr` must point to `len` allocated bytes. +unsafe fn holds_key(ptr: *const u8, len: usize) -> bool { + BUSY.with(|busy| busy.set(true)); + let key = KEY.lock().map(|key| *key).unwrap_or(None); + BUSY.with(|busy| busy.set(false)); + key.is_some_and(|key| find(ptr, len, &key)) +} + +/// [`holds_mark`] for any `needle`. +/// +/// # Safety +/// +/// `ptr` must point to `len` allocated bytes. +unsafe fn find(ptr: *const u8, len: usize, needle: &[u8]) -> bool { + extern "C" { + fn memmem( + haystack: *const c_void, + haystack_len: usize, + needle: *const c_void, + needle_len: usize, + ) -> *mut c_void; + } + !memmem(ptr.cast(), len, needle.as_ptr().cast(), needle.len()).is_null() +} + struct Spy; unsafe impl GlobalAlloc for Spy { @@ -82,7 +126,7 @@ unsafe impl GlobalAlloc for Spy { if watching { // SAFETY: `ptr` is a live block of `layout.size()` bytes until // the `System.dealloc` below. - if holds_mark(ptr, layout.size()) { + if holds_mark(ptr, layout.size()) || holds_key(ptr, layout.size()) { BUSY.with(|busy| busy.set(true)); let trace = std::backtrace::Backtrace::force_capture().to_string(); // Symbol demangling differs by platform: Linux writes @@ -331,3 +375,269 @@ fn the_spy_sees_an_unwiped_free() { let text = marked("control"); watched("control", || drop(text)); } + +/// The JSON index: sealing a document under either root option, opening it +/// whole (from the sealed root, and by walking its chain with the root +/// open), opening a subtree read's rows and an extracted entry, and +/// building a containment needle. Every freed block is checked for the +/// marker, which sits in key names and in string values (one past 128 +/// bytes), and for the data key the document is sealed under, which the key +/// source below records as it hands it out. +/// +/// The input document is built before each watch and the opened values are +/// dropped after it: they are the caller's, not the engine's. +mod json_index { + use super::*; + + use std::borrow::Cow; + + use serde_json::{json, Value}; + use stack_encrypt::json::{ + JsonDocument, JsonNode, JsonOptions, JsonPath, JsonQuery, JsonSubtree, + }; + use stack_encrypt::target::{json_document, AeadContext, Borrowed, CallerContext}; + use stack_encrypt::{op, DecryptFrom, Plan, StackCipher}; + use stack_kms::{ + DataKey, DataKeySource, DataKeyWithTag, DeterministicSource, GenerateKeyPayload, + IdentifiedBy, IndexKey, IndexKeySource, RetrieveKeyPayload, UnverifiedContext, + }; + use uuid::Uuid; + + /// The deterministic source, recording every data key it hands out in + /// [`KEY`] so the spy can look for it. + struct Recorded(DeterministicSource); + + fn record(key: &DataKey) { + *KEY.lock().unwrap() = Some(*key.key()); + } + + impl DataKeySource for Recorded { + async fn generate_keys( + &self, + payloads: Vec>, + keyset_id: Option, + unverified_context: Option>, + ) -> Result, stack_kms::Error> { + let keys = self + .0 + .generate_keys(payloads, keyset_id, unverified_context) + .await?; + keys.iter().for_each(|key| record(&key.key)); + Ok(keys) + } + + async fn retrieve_keys( + &self, + payloads: Vec>, + keyset_id: Option, + unverified_context: Option<&UnverifiedContext>, + ) -> Result, stack_kms::Error> { + let keys = self + .0 + .retrieve_keys(payloads, keyset_id, unverified_context) + .await?; + keys.iter().for_each(record); + Ok(keys) + } + } + + impl IndexKeySource for Recorded { + async fn load_index_key( + &self, + keyset_id: Option, + ) -> Result<(Uuid, IndexKey), stack_kms::Error> { + self.0.load_index_key(keyset_id).await + } + } + + async fn cipher() -> StackCipher { + StackCipher::builder() + .kms(Recorded(DeterministicSource::new([9; 32]))) + .init() + .await + .unwrap() + } + + /// The root option, with the long string and the marked key's `inner` + /// ordered, so the ordering terms over marked text are derived too. + fn options(sealed_root: bool) -> JsonOptions { + let inner = format!("$[\"{}\"].inner", marked("key")); + JsonOptions { + sealed_root, + ..JsonOptions::new() + } + .ordering(["$.long", inner.as_str()]) + .unwrap() + } + + fn context() -> CallerContext { + CallerContext::from(nonempty!("users").with("profile")) + } + + fn aead() -> AeadContext { + AeadContext::from(nonempty!("users").with("profile")) + } + + /// A document with the marker in a key name, in a short string, in a + /// string past 128 bytes, and in an array element under an object. + fn document() -> Value { + let mut long = marked("long"); + long.push_str(&"x".repeat(160)); + json!({ + marked("key"): {"inner": marked("short"), "list": [marked("element"), 1, null]}, + "long": long, + "n": 7, + }) + } + + /// The key a read of the marked key's `inner` member sends, by path. + fn inner_path() -> JsonPath { + JsonPath::parse(&format!("$[\"{}\"].inner", marked("key"))).unwrap() + } + + /// Seal `value` with this thread watched. + fn sealed( + rt: &tokio::runtime::Runtime, + cipher: &StackCipher, + value: &Value, + sealed_root: bool, + ) -> JsonDocument { + // Built before the watch: the ordered paths are the test's input. + let options = options(sealed_root); + watched(&format!("seal (sealed_root: {sealed_root})"), || { + rt.block_on(async { + cipher + .default_keyset() + .run(json_document::<_, Borrowed>(options), value, context()) + .await + .unwrap() + }) + }) + } + + #[test] + fn sealing_and_opening_a_document_leaves_no_plaintext_or_key_on_the_heap() { + let rt = runtime(); + let cipher = rt.block_on(cipher()); + let value = document(); + for sealed_root in [true, false] { + let doc = sealed(&rt, &cipher, &value, sealed_root); + let opened: Value = watched(&format!("open (sealed_root: {sealed_root})"), || { + rt.block_on(async { doc.clone().decrypt_into(&cipher, aead()).await.unwrap() }) + }); + assert_eq!(opened, value); + } + *KEY.lock().unwrap() = None; + } + + #[test] + fn a_read_and_an_extraction_leave_no_plaintext_or_key_on_the_heap() { + let rt = runtime(); + let cipher = rt.block_on(cipher()); + let value = document(); + for sealed_root in [true, false] { + let doc = sealed(&rt, &cipher, &value, sealed_root); + let keyset = cipher.default_keyset(); + let path = inner_path(); + let key = rt + .block_on(async { + keyset + .json_subtree_key( + &path, + &options(sealed_root), + nonempty!("users").with("profile"), + ) + .await + }) + .unwrap(); + // The rows as the SQL returns them, built before the watch. + let rows: Vec = doc + .subtree(&key.token_bytes()) + .iter() + .map(|(j, _, d)| { + let mut wire = serde_json::to_value(doc.node(*j).unwrap()).unwrap(); + wire["d"] = json!(d); + serde_json::from_value(wire).unwrap() + }) + .collect(); + let read = key.open(rows); + let opened: Value = watched( + &format!("subtree read (sealed_root: {sealed_root})"), + || { + rt.block_on(async { + cipher + .decrypt_as::(read, aead()) + .await + .unwrap() + }) + }, + ); + assert_eq!(opened, json!(marked("short"))); + let selector = rt + .block_on(async { + keyset + .json_selector(&path, nonempty!("users").with("profile")) + .await + }) + .unwrap(); + let (j, _) = doc.find(&key.token_bytes()[0]).unwrap(); + let node = selector.open(doc.node(j).unwrap()); + let opened: Value = watched(&format!("-> (sealed_root: {sealed_root})"), || { + rt.block_on(async { + cipher + .decrypt_as::(node, aead()) + .await + .unwrap() + }) + }); + assert_eq!(opened, json!(marked("short"))); + } + *KEY.lock().unwrap() = None; + } + + #[test] + fn a_containment_needle_leaves_no_plaintext_on_the_heap() { + let rt = runtime(); + let cipher = rt.block_on(cipher()); + let plan = Plan::value::() + .context("users/profile") + .encrypt_into::() + .build() + .unwrap(); + let needle = document(); + let query: JsonQuery = watched("Contains needle", || { + rt.block_on(async { cipher.query(op::Contains, &needle).on(&plan).await.unwrap() }) + }); + drop(query); + // The path is a `&'static str`: an owned path `String` is cloned by + // the query builder and dropped by the plan chain, and the chain's + // `P: AsRef` cannot wipe it (see the module's JSON notes). + let path: &'static str = + Box::leak(format!("$[\"{}\"].inner", marked("key")).into_boxed_str()); + let at = (path, marked("short")); + let query: JsonQuery = watched("AtPath(Eq) needle", || { + rt.block_on(async { + cipher + .query(op::AtPath(op::Eq), &at) + .on(&plan) + .await + .unwrap() + }) + }); + drop(query); + } + + /// The spy sees the data key too: a copy of the key a seal was handed, + /// freed while watched, fails the watch. + #[test] + #[should_panic(expected = "freed with plaintext in them")] + fn the_spy_sees_a_freed_data_key() { + let rt = runtime(); + let cipher = rt.block_on(cipher()); + let value = document(); + let _ = sealed(&rt, &cipher, &value, true); + let key = KEY.lock().unwrap().expect("the seal recorded its key"); + let copy = key.to_vec(); + watched("control (key)", || drop(copy)); + } +} From c18f2947aa91961e42a19563c824305a240719a3 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 17:57:07 +1100 Subject: [PATCH 11/22] refactor(stack-encrypt): draw the JSON index's permutations with rand's shuffle from one source MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The JSON index shuffled a document's stored order and a query's needle with a Fisher–Yates loop of its own, drew a query's order from a seed the terms carried, and kept its deterministic test source as a production enum. `json::order` is now `json::shuffle`: the shuffle is `rand::seq::SliceRandom::shuffle` driven by `SafeRand` (the same rand vitaminc-random wraps, so the generator is unchanged), the source is one `Randomness` struct whose seeded stream exists only under `test-support` and seeds each document's or query's generator with `SeedableRng::from_rng`, and a needle's order is drawn when its terms are derived, so clones of one terms value share it and nothing carries a seed. vitaminc-permutation was considered: its `PermutationKey` is defined for N of 8, 16, 32, 64 and 128 only, and a document has any number of nodes. The pinned vectors move in `order` alone: the tags and tokens are unchanged. Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- Cargo.lock | 1 + packages/stack-encrypt/Cargo.toml | 8 +- packages/stack-encrypt/src/cipher.rs | 63 ++++--- packages/stack-encrypt/src/json/derive.rs | 64 +++---- packages/stack-encrypt/src/json/mod.rs | 3 +- packages/stack-encrypt/src/json/order.rs | 173 ------------------ packages/stack-encrypt/src/json/query.rs | 42 ++--- packages/stack-encrypt/src/json/shuffle.rs | 157 ++++++++++++++++ packages/stack-encrypt/tests/common/mod.rs | 9 +- .../tests/fixtures/json_tree_vectors.json | 68 +++---- .../stack-encrypt/tests/json_tree_vectors.rs | 4 +- 11 files changed, 294 insertions(+), 298 deletions(-) delete mode 100644 packages/stack-encrypt/src/json/order.rs create mode 100644 packages/stack-encrypt/src/json/shuffle.rs diff --git a/Cargo.lock b/Cargo.lock index a77b94a88..542638e5e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3407,6 +3407,7 @@ dependencies = [ "miette", "ore-rs", "proptest", + "rand 0.10.1", "rust_decimal", "serde", "serde_json", diff --git a/packages/stack-encrypt/Cargo.toml b/packages/stack-encrypt/Cargo.toml index 598cdaa5b..4cfa09dc0 100644 --- a/packages/stack-encrypt/Cargo.toml +++ b/packages/stack-encrypt/Cargo.toml @@ -42,8 +42,14 @@ vitaminc-normalize = { workspace = true } vitaminc-ore = { workspace = true } vitaminc-prf = { workspace = true } vitaminc-protected = { workspace = true } -# The generator that draws a JSON document's stored order (`json::order`). +# The generator the JSON index draws its permutations from (`json::shuffle`): +# `vitaminc::random::SafeRand`, seeded from the operating system. vitaminc = { workspace = true } +# `rand::seq::SliceRandom::shuffle`, the Fisher–Yates shuffle `SafeRand` +# drives. The version is vitaminc-random's, so the `Rng` it asks for is the +# one `SafeRand` implements. No default features: `std`, the OS and thread +# generators are not used. +rand = { version = "0.10.1", default-features = false } # `dynamic` only: serde over the transport frame, which reads a binding's # control data (statistics, options, a plan) into typed messages. vitaminc-aead-frame = { workspace = true, optional = true, features = ["serde"] } diff --git a/packages/stack-encrypt/src/cipher.rs b/packages/stack-encrypt/src/cipher.rs index 7419f7292..dc6473c5b 100644 --- a/packages/stack-encrypt/src/cipher.rs +++ b/packages/stack-encrypt/src/cipher.rs @@ -531,9 +531,10 @@ pub struct StackCipher { /// Every other keyset this cipher has selected, least recently used /// first out. See [`keyset`](Self::keyset). keysets: Mutex, - /// Where the stored orders of the JSON documents this cipher seals are - /// drawn from: fresh entropy per document, outside tests. - json_order: crate::json::order::OrderSource, + /// Where the JSON index draws its permutations from (a document's stored + /// order, a query's needle order): fresh entropy per document or query, + /// outside tests. + json_randomness: crate::json::Randomness, /// The backend's [`IndexKeySource::load_index_key`], captured at /// [`init`](StackCipherBuilder::init), where `K` is known to implement /// it. Settling a [`Pending`](crate::target::Pending) asks only for a @@ -653,9 +654,9 @@ impl StackCipher { Ok(self.keysets().insert(state, resolution)) } - /// Where this cipher draws a JSON document's stored order from. - pub(crate) fn json_order(&self) -> &crate::json::order::OrderSource { - &self.json_order + /// Where this cipher draws the JSON index's permutations from. + pub(crate) fn json_randomness(&self) -> &crate::json::Randomness { + &self.json_randomness } fn keysets(&self) -> std::sync::MutexGuard<'_, KeysetCache> { @@ -843,7 +844,8 @@ pub struct StackCipherBuilder { kms: K, cache_size: NonZeroUsize, name_ttl: Duration, - json_order_seed: Option>, + #[cfg(feature = "test-support")] + json_seed: Option>, } impl StackCipherBuilder { @@ -857,7 +859,8 @@ impl StackCipherBuilder { kms: FromEnv, cache_size: KeysetCache::DEFAULT_CAPACITY, name_ttl: DEFAULT_NAME_TTL, - json_order_seed: None, + #[cfg(feature = "test-support")] + json_seed: None, } } } @@ -893,19 +896,19 @@ impl StackCipherBuilder { self } - /// Test support: draw the stored order of every JSON document the - /// cipher seals from one generator started at `seed`, so that a run - /// sealing the same documents in the same sequence stores them in the - /// same orders, and a fixture can pin them. What the deterministic key - /// source (`kms::DeterministicSource`) is paired with. + /// Test support: draw every permutation the JSON index needs (the + /// stored order of each document it seals, the order of each query's + /// needle) from one generator started at `seed`, + /// so that a run sealing the same documents in the same sequence draws + /// the same permutations and a fixture can pin them. Paired with the + /// deterministic key source (`kms::DeterministicSource`). /// - /// Never for data: a document's stored order is drawn from the - /// operating system's entropy precisely so that it says nothing, and an - /// order that follows from a known seed hides nothing of the document's - /// shape (see `json::order`). + /// Never for data: the permutations are drawn from the operating + /// system's entropy so that they say nothing, and one that follows from + /// a known seed hides nothing of a document's shape. #[cfg(feature = "test-support")] - pub fn json_order_seed(mut self, seed: [u8; 32]) -> Self { - self.json_order_seed = Some(Protected::new(seed)); + pub fn json_seed(mut self, seed: [u8; 32]) -> Self { + self.json_seed = Some(Protected::new(seed)); self } } @@ -923,7 +926,8 @@ impl StackCipherBuilder { kms, cache_size: self.cache_size, name_ttl: self.name_ttl, - json_order_seed: self.json_order_seed, + #[cfg(feature = "test-support")] + json_seed: self.json_seed, } } @@ -951,6 +955,18 @@ impl StackCipherBuilder { } } +impl StackCipherBuilder { + /// The randomness the cipher's JSON index draws from: the seeded stream + /// under `test-support` when one was given, entropy otherwise. + fn json_randomness(&self) -> crate::json::Randomness { + #[cfg(feature = "test-support")] + if let Some(seed) = &self.json_seed { + return crate::json::Randomness::seeded(seed.clone()); + } + crate::json::Randomness::from_entropy() + } +} + impl StackCipherBuilder { /// Resolve the default keyset and load its index key, producing a /// cipher whose [`default_keyset`](StackCipher::default_keyset) can both @@ -958,6 +974,7 @@ impl StackCipherBuilder { /// always pays; every other keyset loads on first selection. pub async fn init(self) -> Result, Error> { let default = load_default_keyset(&self.kms).await?; + let json_randomness = self.json_randomness(); Ok(StackCipher { kms: self.kms, keysets: Mutex::new(KeysetCache::new( @@ -966,11 +983,7 @@ impl StackCipherBuilder { Arc::clone(&default), )), default, - json_order: self - .json_order_seed - .map_or(crate::json::order::OrderSource::Entropy, |seed| { - crate::json::order::OrderSource::controlled(seed) - }), + json_randomness, index_keys: load_index_key_of::, }) } diff --git a/packages/stack-encrypt/src/json/derive.rs b/packages/stack-encrypt/src/json/derive.rs index 22f3f1435..185ddc6fd 100644 --- a/packages/stack-encrypt/src/json/derive.rs +++ b/packages/stack-encrypt/src/json/derive.rs @@ -23,13 +23,13 @@ use vitaminc_protected::{AsProtectedRef, Controlled, Protected}; use zeroize::{Zeroize, Zeroizing}; use super::cascade::{self, ChildrenDigest}; -use super::order; use super::path::{key_step_len, push_key, JsonPath, Segment, ARRAY_STEP}; use super::plain::{ insert_member, protected_array, protected_object, push_element, read_protected, take_protected, PlainJson, }; -use super::query::{JsonSelector, JsonSubtreeKey, Read}; +use super::query::{group_of, JsonProjection, JsonSelector, JsonSubtreeKey, Read}; +use super::shuffle; use super::{JsonDocument, JsonEntry, JsonExtracted, JsonHeader, JsonOptions, JsonSubtree, Tag}; use crate::keyset::KeysetState; use crate::sem::TermError; @@ -218,12 +218,11 @@ pub struct JsonTerms { /// scalar has as an element of a root array, so a containment needle /// can match a stored `["a"]` as `jsonb @>` does. alt: Option<(Tag, Tag)>, - /// The seed of the order a needle built from these terms is sent in, - /// drawn from the cipher's [`OrderSource`](super::order::OrderSource) - /// when the terms are derived for a query ([`terms_pending`]); `None` - /// for the terms a document is sealed from, which no needle is built - /// from. A seed decides a permutation of public pairs and nothing else. - needle_order: Option>, + /// The projection a containment query built from these terms sends, + /// its pair lists and element groups in a random order drawn when the + /// terms were derived for a query ([`terms_pending`]). `None` for the + /// terms a document is sealed from, which no query is built from. + needle: Option, } /// The key names and the tokens are what the terms must not leak, so the @@ -257,6 +256,7 @@ impl JsonTerms { self.nodes.is_empty() } + #[cfg(test)] pub(crate) fn nodes(&self) -> &[TermNode] { &self.nodes } @@ -265,12 +265,11 @@ impl JsonTerms { self.alt } - /// The generator a needle built from these terms is shuffled with, if - /// the terms were derived for a query. - pub(crate) fn needle_order(&self) -> Option { - self.needle_order - .clone() - .map(SafeRand::from_controlled_seed) + /// The projection a containment query sends: the one drawn when the + /// terms were derived for a query, or, for terms no query was derived + /// for, the nodes' projection in the walk's order. + pub(crate) fn into_needle(self) -> JsonProjection { + self.needle.unwrap_or_else(|| group_of(&self.nodes, None)) } /// The last node of the walk: in a needle built from one path, the @@ -615,7 +614,7 @@ fn walk( terms: JsonTerms { nodes: walker.nodes, alt, - needle_order: None, + needle: None, }, plain: walker.plain, }) @@ -769,7 +768,11 @@ fn shuffled_ordinals<'m>( // ============================================================================= /// The one-way terms of `value` under `context`, as the index's operation -/// derives them: no key is requested. +/// derives them for a query: no key is requested. The projection a +/// containment query sends is drawn here, in a random order of its own +/// ([`shuffle`]), so a needle shows no more of the order of its keys than a +/// stored `q` does; `jsonb @>` reads both as sets, so the order changes no +/// answer. pub(crate) fn terms_pending<'a, K>( value: &Value, options: &JsonOptions, @@ -779,14 +782,13 @@ pub(crate) fn terms_pending<'a, K>( let context = context.into_prf_context().into_owned(); let terms = walk(value, options, cipher.prf(), &context) .map_err(Error::from) - .and_then(|w| { - // A needle is sent in a random order of its own, drawn as a - // document's stored order is ([`order`]): fresh entropy per - // query outside tests. - let mut order = cipher.cipher().json_order().document()?; + .and_then(|walked| { + let mut rng = cipher.cipher().json_randomness().generator()?; + let mut needle = group_of(&walked.terms.nodes, None); + needle.shuffle(&mut rng); Ok(JsonTerms { - needle_order: Some(order::seed(&mut order)), - ..w.terms + needle: Some(needle), + ..walked.terms }) }); Pending::ready(cipher, terms) @@ -825,8 +827,8 @@ pub(crate) fn seal_pending<'a, K: 'static>( }; let keyset_id = cipher.keyset_id(); let sealed_root = options.sealed_root; - let order = match cipher.cipher().json_order().document() { - Ok(order) => order, + let rng = match cipher.cipher().json_randomness().generator() { + Ok(rng) => rng, Err(error) => return Pending::failed(cipher, error), }; Pending::request( @@ -834,7 +836,7 @@ pub(crate) fn seal_pending<'a, K: 'static>( vec![Request::generate_under(descriptor)], move |responses| { let key = responses.next_generated_key()?; - seal(walked, whole, keyset_id, key, &aad, sealed_root, order) + seal(walked, whole, keyset_id, key, &aad, sealed_root, rng) }, ) } @@ -986,7 +988,7 @@ fn sealed_root_plaintext(value: &Value) -> Result>, serde_json } /// Seal the walked nodes under `key`, from the leaves up, each at the slot -/// `order` draws for it ([`order::slots`]), and shuffle the projection with +/// `rng` draws for it ([`shuffle::slots`]), and shuffle the projection with /// the same generator. fn seal( walked: Walked, @@ -995,7 +997,7 @@ fn seal( key: stack_kms::DataKeyWithTag, aad: &Context<'_>, sealed_root: bool, - mut order: SafeRand, + mut rng: SafeRand, ) -> Result { if key.tag.len() > usize::from(u16::MAX) { return Err(Error::Other( @@ -1012,7 +1014,7 @@ fn seal( // root commits to no children and opens under the empty digest. let children = cascade::children_of(nodes.iter().map(|n| n.parent.zip(n.ordinal))); // Where each node is stored, by its index in the walk; the AAD binds it. - let slots = order::slots(nodes.len(), &mut order); + let slots = shuffle::slots(nodes.len(), &mut rng); let mut sealed: Vec> = vec![Vec::new(); nodes.len()]; // From the leaves up: the walk is pre-order, so every descendant of a // node has a larger index, and the reverse order seals a node's children @@ -1051,8 +1053,8 @@ fn seal( } // `slots` is a permutation, so every slot was filled once. let entries: Vec = entries.into_iter().flatten().collect(); - let mut projection = super::query::group_of(nodes, None); - projection.shuffle(&mut order); + let mut projection = group_of(nodes, None); + projection.shuffle(&mut rng); Ok(JsonDocument::new( JsonHeader::new(keyset_id, iv, tag, sealed_root), entries, diff --git a/packages/stack-encrypt/src/json/mod.rs b/packages/stack-encrypt/src/json/mod.rs index 3ea0f880e..d17ffe839 100644 --- a/packages/stack-encrypt/src/json/mod.rs +++ b/packages/stack-encrypt/src/json/mod.rs @@ -124,10 +124,10 @@ use crate::target::{DecryptField, DecryptInto, Decryptable, Decryption}; mod cascade; mod derive; -pub(crate) mod order; mod path; mod plain; mod query; +mod shuffle; pub use derive::JsonTerms; pub use path::{JsonPath, JsonPathError, Segment}; @@ -136,6 +136,7 @@ pub use query::{ }; pub(crate) use derive::{seal_pending, terms_pending, wipe_value}; +pub(crate) use shuffle::Randomness; /// A sixteen-byte tag: a selector or a token. pub type Tag = [u8; 16]; diff --git a/packages/stack-encrypt/src/json/order.rs b/packages/stack-encrypt/src/json/order.rs deleted file mode 100644 index 5554a27a4..000000000 --- a/packages/stack-encrypt/src/json/order.rs +++ /dev/null @@ -1,173 +0,0 @@ -//! The stored order: where a document's entries sit in `sv`, and the order -//! of its containment projection `q`. -//! -//! A document's entries are stored in a uniformly random order, drawn -//! afresh for each document when it is sealed and then thrown away, with -//! the root at slot `0`, where the sealed-root fallback finds it. Every -//! document has a root, so its slot says nothing. The pairs of each group of -//! `q`, and the element groups of each list, are shuffled the same way. -//! -//! Nothing needs the permutation afterwards. The server finds entries by -//! their self and parent tags and reads `q` as sets (`jsonb @>`), and the -//! client places a read's rows by the chain and verifies them by the cascade -//! ([`cascade`](super::cascade)); the AAD binds each entry's slot, so an -//! entry cannot be moved between slots, but nothing reads a slot as a -//! position in the document. So the permutation is not derived from -//! anything, keyed or not: whatever it were derived from would have to be -//! kept from the database, and nothing is gained by being able to derive -//! it. -//! -//! What the depth-first order used to show and this one does not: each -//! row's tree (the parent of every node), the size of every subtree, and -//! which of two sibling key names sorts first. What any order leaves -//! visible is listed on the leakage page -//! (`packages/eql/docs/reference/json-tree-leakage.md`). -//! -//! The generator is `vitaminc`'s [`SafeRand`] (ChaCha20, seeded from the -//! operating system) and each draw is [`SafeRand::next_below`] (Lemire's -//! reduction: one 64-bit draw per call, bias at most `n / 2⁶⁴`, negligible -//! for any document). Which generator draws the permutation is not part of -//! the format, since nothing stored depends on it, and it can be replaced -//! without a version change. - -use std::sync::Mutex; - -use vitaminc::random::{Rng, SafeRand}; -use vitaminc_protected::{Controlled, Protected}; - -use crate::Error; - -/// Where a cipher draws the stored orders of the documents it seals from. -pub(crate) enum OrderSource { - /// A fresh generator per document, seeded from the operating system: - /// the only source outside tests. - Entropy, - /// A generator per document seeded from one stream that starts at a - /// fixed seed, so a run that seals the same documents in the same order - /// stores them in the same orders. Test support only - /// ([`StackCipherBuilder::json_order_seed`](crate::StackCipherBuilder)). - Controlled(Box>), -} - -impl OrderSource { - /// The source a fixed `seed` starts. - pub(crate) fn controlled(seed: Protected<[u8; 32]>) -> Self { - Self::Controlled(Box::new(Mutex::new(SafeRand::from_controlled_seed(seed)))) - } - - /// The generator one document's order is drawn from. - pub(crate) fn document(&self) -> Result { - match self { - Self::Entropy => SafeRand::from_entropy() - .map_err(|_| Error::Other("no entropy for a JSON document's stored order".into())), - Self::Controlled(stream) => { - // The stream holds no invariant a panic could break. - let mut stream = stream - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - Ok(SafeRand::from_controlled_seed(seed(&mut stream))) - } - } - } -} - -/// Thirty-two bytes drawn from `rng`, as the seed of another generator: -/// one document's order from a controlled stream, or the order a query's -/// needle is sent in, which its terms carry until the needle is built -/// ([`JsonTerms`](super::JsonTerms)). -pub(crate) fn seed(rng: &mut SafeRand) -> Protected<[u8; 32]> { - Protected::generate(|| { - let mut seed = [0u8; 32]; - rng.fill_bytes(&mut seed); - seed - }) -} - -/// A uniform shuffle of `items` (Fisher–Yates), each swap index drawn with -/// [`SafeRand::next_below`]. -pub(crate) fn shuffle(items: &mut [T], rng: &mut SafeRand) { - for i in (1..items.len()).rev() { - // `items` is a document's entries, or a group of its projection: - // never more than `u32::MAX` of them, since an ordinal past - // PostgreSQL's integer is refused when the document is walked. - let bound = u32::try_from(i + 1).unwrap_or(u32::MAX); - let j = rng.next_below(bound) as usize; - items.swap(i, j); - } -} - -/// The slot each of `n` nodes is stored at, by its index in the walk: the -/// root (index `0`) at slot `0`, and the other nodes at a uniformly random -/// permutation of the slots `1..n`. -pub(crate) fn slots(n: usize, rng: &mut SafeRand) -> Vec { - let mut slots: Vec = (0..n).collect(); - if let Some(rest) = slots.get_mut(1..) { - shuffle(rest, rng); - } - slots -} - -#[cfg(test)] -mod tests { - use super::*; - - fn seeded(byte: u8) -> SafeRand { - SafeRand::from_controlled_seed(Protected::new([byte; 32])) - } - - #[test] - fn the_slots_are_a_permutation_with_the_root_first() { - for n in [0, 1, 2, 3, 17, 200] { - let slots = slots(n, &mut seeded(1)); - let mut sorted = slots.clone(); - sorted.sort_unstable(); - assert_eq!(sorted, (0..n).collect::>(), "{n}: a permutation"); - if n > 0 { - assert_eq!(slots[0], 0, "{n}: the root at slot 0"); - } - } - assert_ne!( - slots(50, &mut seeded(1)), - (0..50).collect::>(), - "not the identity (with overwhelming probability)" - ); - } - - /// Every arrangement of three items comes up, each about a sixth of the - /// time: the shuffle reaches every permutation and favours none. - #[test] - fn the_shuffle_is_uniform_over_small_permutations() { - let mut rng = seeded(2); - let mut counts = std::collections::BTreeMap::new(); - let runs = 60_000; - for _ in 0..runs { - let mut items = [0, 1, 2]; - shuffle(&mut items, &mut rng); - *counts.entry(items).or_insert(0usize) += 1; - } - assert_eq!(counts.len(), 6, "every permutation of three"); - for (perm, count) in counts { - assert!( - (9_000..11_000).contains(&count), - "{perm:?} came up {count} times in {runs}" - ); - } - } - - #[test] - fn a_controlled_source_repeats_and_entropy_does_not() { - let order = |source: &OrderSource| { - let mut a = source.document().unwrap(); - let mut b = source.document().unwrap(); - (slots(30, &mut a), slots(30, &mut b)) - }; - let one = order(&OrderSource::controlled(Protected::new([3; 32]))); - let two = order(&OrderSource::controlled(Protected::new([3; 32]))); - assert_eq!(one, two, "the same seed, the same orders"); - assert_ne!(one.0, one.1, "each document draws its own order"); - let other = order(&OrderSource::controlled(Protected::new([4; 32]))); - assert_ne!(one, other, "another seed, other orders"); - let fresh = order(&OrderSource::Entropy); - assert_ne!(fresh.0, fresh.1, "entropy draws a new order per document"); - } -} diff --git a/packages/stack-encrypt/src/json/query.rs b/packages/stack-encrypt/src/json/query.rs index 755ef8381..d63434092 100644 --- a/packages/stack-encrypt/src/json/query.rs +++ b/packages/stack-encrypt/src/json/query.rs @@ -77,13 +77,13 @@ impl JsonProjection { } /// Put the pairs of every group, and the groups of every element list, - /// in a uniformly random order ([`order::shuffle`](super::order::shuffle)). + /// in a uniformly random order ([`shuffle`](super::shuffle::shuffle)). /// The server reads `q` as sets (`jsonb @>` over `sv` and `el`), so the /// order changes no answer; shuffled, it no longer restates the /// document's depth-first order. pub(crate) fn shuffle(&mut self, rng: &mut vitaminc::random::SafeRand) { - super::order::shuffle(&mut self.sv, rng); - super::order::shuffle(&mut self.el, rng); + super::shuffle::shuffle(&mut self.sv, rng); + super::shuffle::shuffle(&mut self.el, rng); for group in &mut self.el { group.shuffle(rng); } @@ -110,13 +110,14 @@ impl JsonProjection { /// operand: `{"sv": [{"s": "…", "vs": "…"}, …], "el": [{"sv": …, "el": …}, /// …], "alt": {…}}`. /// -/// The projection is sent in a random order of its own, drawn for each -/// query as a stored `q`'s order is drawn for each document: every pair -/// list and element-group list shuffled, recursively. `jsonb @>` is set +/// The projection is sent in a random order of its own, drawn when the +/// terms were derived ([`terms_pending`](super::derive::terms_pending)) as a +/// stored `q`'s order is drawn when its document is sealed: every pair list +/// and element-group list shuffled, recursively. `jsonb @>` is set /// containment over arrays, so the order changes no answer, and a needle /// shows no more of the order of its keys than a stored row does. Two -/// queries for one value therefore serialize differently and compare equal -/// (equality is as sets). +/// derivations of one value therefore serialize differently and compare +/// equal (equality is as sets). #[derive(Clone, Debug, PartialEq, Eq)] pub struct JsonQuery { group: JsonProjection, @@ -128,20 +129,20 @@ impl JsonQuery { /// the value at the path, with no `alt`. The alternative matches a root /// array that holds a root scalar, which is containment; at a path, /// `("$", 1)` asks for the scalar `1`, not for an array holding it. - pub(crate) fn at_path(terms: &JsonTerms) -> Self { + pub(crate) fn at_path(terms: JsonTerms) -> Self { Self { - group: needle(terms), + group: terms.into_needle(), alt: None, } } - pub(crate) fn of_terms(terms: &JsonTerms) -> Self { + pub(crate) fn of_terms(terms: JsonTerms) -> Self { let alt = terms.alt().map(|(s, vs)| JsonProjection { sv: vec![Pair16 { s, vs }], el: Vec::new(), }); Self { - group: needle(terms), + group: terms.into_needle(), alt, } } @@ -160,19 +161,6 @@ impl JsonQuery { } } -/// The projection a needle sends: the terms' group, its pair lists and -/// element groups shuffled, recursively, with the generator the terms carry -/// ([`JsonProjection::shuffle`]), as a stored `q` is shuffled. `jsonb @>` -/// is set containment over arrays, so the order changes no answer; in the -/// walk's order a needle would show which of two sibling keys sorts first. -fn needle(terms: &JsonTerms) -> JsonProjection { - let mut group = group_of(terms.nodes(), None); - if let Some(mut order) = terms.needle_order() { - group.shuffle(&mut order); - } - group -} - /// The group of the nodes that sit directly in `parent` (`None` is the top /// level), with the container elements in it nested as groups. What a /// needle is, and what a document writes to `q` when it is sealed. @@ -689,7 +677,7 @@ macro_rules! containment { } fn verifier(&self, _: &Value) {} fn query((): (), terms: JsonTerms, _: QueryToken) -> JsonQuery { - JsonQuery::of_terms(&terms) + JsonQuery::of_terms(terms) } } impl Answers<$op, Value> for JsonDocument { @@ -722,7 +710,7 @@ where } fn verifier(&self, _: &(P, V)) {} fn query((): (), terms: AtPathTerms, _: QueryToken) -> JsonQuery { - JsonQuery::at_path(&terms.terms) + JsonQuery::at_path(terms.terms) } } impl Answers, (P, V)> for JsonDocument diff --git a/packages/stack-encrypt/src/json/shuffle.rs b/packages/stack-encrypt/src/json/shuffle.rs new file mode 100644 index 000000000..24d17f265 --- /dev/null +++ b/packages/stack-encrypt/src/json/shuffle.rs @@ -0,0 +1,157 @@ +//! The JSON index's random permutations: the slots a document's entries are +//! stored at, the order of the pairs and groups of a stored projection `q`, +//! and the order a containment query sends its selectors in. +//! +//! Each permutation is uniformly random, drawn when the document is sealed +//! or the query derived, and then thrown away. Nothing reads one back: the +//! server finds entries by their tags and reads `q` as sets (`jsonb @>`), +//! and the client places a read's rows by the chain and checks them by the +//! cascade ([`cascade`](super::cascade)). So a permutation is derived from +//! nothing, keyed or not: anything it were derived from would have to be +//! kept from the database, and nothing is gained by being able to recompute +//! it. What the permutations hide, and what no order can hide, is on the +//! leakage page (`packages/eql/docs/reference/json-tree-leakage.md`). +//! +//! The draws come from vitaminc's [`SafeRand`] (ChaCha20, seeded from the +//! operating system for each document or query) through rand's +//! [`shuffle`](SliceRandom::shuffle), a Fisher–Yates shuffle with unbiased +//! index draws. The generator is not part of the format, since nothing +//! stored depends on it. + +use rand::seq::SliceRandom; +use vitaminc::random::SafeRand; + +use crate::Error; + +/// Where the JSON index draws its permutations from: a fresh generator for +/// each document sealed or query derived, seeded from the operating system. +/// +/// With the `test-support` feature a cipher can instead be given one seeded +/// stream ([`StackCipherBuilder::json_seed`](crate::StackCipherBuilder)), +/// which every generator is then seeded from, so that a run sealing the +/// same documents in the same sequence draws the same permutations and a +/// fixture can pin them. +pub(crate) struct Randomness { + #[cfg(any(test, feature = "test-support"))] + stream: Option>, +} + +impl Randomness { + /// Every generator seeded from the operating system. + pub(crate) const fn from_entropy() -> Self { + Self { + #[cfg(any(test, feature = "test-support"))] + stream: None, + } + } + + /// Every generator seeded from one stream that starts at `seed`. + #[cfg(any(test, feature = "test-support"))] + pub(crate) fn seeded(seed: vitaminc_protected::Protected<[u8; 32]>) -> Self { + Self { + stream: Some(std::sync::Mutex::new(SafeRand::from_controlled_seed(seed))), + } + } + + /// A generator for one document or one query. + pub(crate) fn generator(&self) -> Result { + #[cfg(any(test, feature = "test-support"))] + if let Some(stream) = &self.stream { + use vitaminc::random::SeedableRng; + // The stream holds no invariant a panic could break. + let mut stream = stream + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + return Ok(SafeRand::from_rng(&mut *stream)); + } + SafeRand::from_entropy() + .map_err(|_| Error::Other("no entropy for the JSON index's permutations".into())) + } +} + +/// `items` in a uniformly random order. +pub(crate) fn shuffle(items: &mut [T], rng: &mut SafeRand) { + items.shuffle(rng); +} + +/// The slot each of `n` nodes is stored at, by its index in the walk: the +/// root (index `0`) at slot `0`, where the sealed-root fallback finds it, +/// and the other nodes at a uniformly random permutation of the slots +/// `1..n`. +pub(crate) fn slots(n: usize, rng: &mut SafeRand) -> Vec { + let mut slots: Vec = (0..n).collect(); + if let Some(rest) = slots.get_mut(1..) { + shuffle(rest, rng); + } + slots +} + +#[cfg(test)] +mod tests { + use super::*; + use vitaminc_protected::Protected; + + fn seeded(byte: u8) -> SafeRand { + SafeRand::from_controlled_seed(Protected::new([byte; 32])) + } + + fn is_permutation(items: &[usize], n: usize) -> bool { + let mut sorted = items.to_vec(); + sorted.sort_unstable(); + sorted == (0..n).collect::>() + } + + #[test] + fn the_slots_are_a_permutation_with_the_root_first() { + for n in [0, 1, 2, 3, 17, 200] { + let slots = slots(n, &mut seeded(1)); + assert!(is_permutation(&slots, n), "{n}: a permutation"); + if n > 0 { + assert_eq!(slots[0], 0, "{n}: the root at slot 0"); + } + } + assert_ne!( + slots(50, &mut seeded(1)), + (0..50).collect::>(), + "not the identity (with overwhelming probability)" + ); + } + + /// Every arrangement of three items comes up, each about a sixth of the + /// time: the shuffle reaches every permutation and favours none. + #[test] + fn the_shuffle_is_uniform_over_small_permutations() { + let mut rng = seeded(2); + let mut counts = std::collections::BTreeMap::new(); + let runs = 60_000; + for _ in 0..runs { + let mut items = [0, 1, 2]; + shuffle(&mut items, &mut rng); + *counts.entry(items).or_insert(0usize) += 1; + } + assert_eq!(counts.len(), 6, "every permutation of three"); + for (perm, count) in counts { + assert!( + (9_000..11_000).contains(&count), + "{perm:?} came up {count} times in {runs}" + ); + } + } + + #[test] + fn a_seeded_source_repeats_and_entropy_does_not() { + let order = |source: &Randomness| { + let mut a = source.generator().unwrap(); + let mut b = source.generator().unwrap(); + (slots(30, &mut a), slots(30, &mut b)) + }; + let one = order(&Randomness::seeded(Protected::new([3; 32]))); + let two = order(&Randomness::seeded(Protected::new([3; 32]))); + assert_eq!(one, two, "the same seed, the same permutations"); + assert_ne!(one.0, one.1, "each document draws its own"); + let other = order(&Randomness::seeded(Protected::new([4; 32]))); + assert_ne!(one, other, "another seed, other permutations"); + let fresh = order(&Randomness::from_entropy()); + assert_ne!(fresh.0, fresh.1, "entropy draws a new one per document"); + } +} diff --git a/packages/stack-encrypt/tests/common/mod.rs b/packages/stack-encrypt/tests/common/mod.rs index c99cdc37b..e761ea8a1 100644 --- a/packages/stack-encrypt/tests/common/mod.rs +++ b/packages/stack-encrypt/tests/common/mod.rs @@ -226,13 +226,14 @@ pub async fn recording_cipher() -> (StackCipher, Arc StackCipher { let builder = StackCipher::builder().kms(DeterministicSource::new(seed)); #[cfg(feature = "test-support")] - let builder = builder.json_order_seed(seed); + let builder = builder.json_seed(seed); builder.init().await.expect("build cipher") } diff --git a/packages/stack-encrypt/tests/fixtures/json_tree_vectors.json b/packages/stack-encrypt/tests/fixtures/json_tree_vectors.json index 38207bf2f..74f6c8688 100644 --- a/packages/stack-encrypt/tests/fixtures/json_tree_vectors.json +++ b/packages/stack-encrypt/tests/fixtures/json_tree_vectors.json @@ -18,25 +18,15 @@ "s": "GUG6+MOjeHGk8vJS5CJOxA==", "x": "PelL2/QWv0M=" }, - { - "p": "WN3/UZRAros=", - "s": "HlQhfaUoEj5O+JR/j9trVQ==", - "x": "W0aJLUxlVdA=" - }, - { - "p": "sIWG0EdV0i8=", - "s": "Ie6Xm6ZxkV/9nBoIK7zXsQ==", - "x": "HRuqzbdL4ew=" - }, { "p": "8OU5UZoL5Hw=", "s": "8OVeBxReEgPVrjL7qGYAbg==", "x": "p7/wbsjaDjQ=" }, { - "p": "/EJ22V9WvdY=", - "s": "z1M7JTZ8C8WMd3dqhuZpfA==", - "x": "hMhl3B9cj/0=" + "p": "WN3/UZRAros=", + "s": "HlQhfaUoEj5O+JR/j9trVQ==", + "x": "W0aJLUxlVdA=" }, { "p": "ZHxAvDKdFsI=", @@ -47,17 +37,27 @@ "p": "ZOSBKJZJ/NY=", "s": "0AOmEo3aMm7GHQ6VNNv+Vw==", "x": "EVNlMjX0dyE=" + }, + { + "p": "/EJ22V9WvdY=", + "s": "z1M7JTZ8C8WMd3dqhuZpfA==", + "x": "hMhl3B9cj/0=" + }, + { + "p": "sIWG0EdV0i8=", + "s": "Ie6Xm6ZxkV/9nBoIK7zXsQ==", + "x": "HRuqzbdL4ew=" } ], "iv": "OtEqUO60j5CMut+3HQzYSA==", "order": [ "$", - "$.a.b", - "$.d", "$.a", - "$.a.b[1].c", + "$.a.b", "$.a.b[1]", - "$.a.b[0]" + "$.a.b[0]", + "$.a.b[1].c", + "$.d" ], "q": { "el": [ @@ -135,25 +135,15 @@ "s": "GUG6+MOjeHGk8vJS5CJOxA==", "x": "PelL2/QWv0M=" }, - { - "p": "Dy1WgDveXQU=", - "s": "HlQhfaUoEj5O+JR/j9trVQ==", - "x": "12lMeBeDMpo=" - }, - { - "p": null, - "s": "Ie6Xm6ZxkV/9nBoIK7zXsQ==", - "x": "uGnxy4JLUhk=" - }, { "p": null, "s": "8OVeBxReEgPVrjL7qGYAbg==", "x": "wl8ZJzvRwJ0=" }, { - "p": "4mV5CHfjZEg=", - "s": "z1M7JTZ8C8WMd3dqhuZpfA==", - "x": "A/QZeMRJngk=" + "p": "Dy1WgDveXQU=", + "s": "HlQhfaUoEj5O+JR/j9trVQ==", + "x": "12lMeBeDMpo=" }, { "p": "d0RUZY7M9PI=", @@ -164,17 +154,27 @@ "p": "sJXI7nmR44s=", "s": "0AOmEo3aMm7GHQ6VNNv+Vw==", "x": "/yUSe3uTE98=" + }, + { + "p": "4mV5CHfjZEg=", + "s": "z1M7JTZ8C8WMd3dqhuZpfA==", + "x": "A/QZeMRJngk=" + }, + { + "p": null, + "s": "Ie6Xm6ZxkV/9nBoIK7zXsQ==", + "x": "uGnxy4JLUhk=" } ], "iv": "OtEqUO60j5CMut+3HQzYSA==", "order": [ "$", - "$.a.b", - "$.d", "$.a", - "$.a.b[1].c", + "$.a.b", "$.a.b[1]", - "$.a.b[0]" + "$.a.b[0]", + "$.a.b[1].c", + "$.d" ], "q": { "el": [ diff --git a/packages/stack-encrypt/tests/json_tree_vectors.rs b/packages/stack-encrypt/tests/json_tree_vectors.rs index 0eb5cdc1a..58efc6be2 100644 --- a/packages/stack-encrypt/tests/json_tree_vectors.rs +++ b/packages/stack-encrypt/tests/json_tree_vectors.rs @@ -13,7 +13,7 @@ //! //! Sealed under the deterministic key source, whose index key is fixed and //! whose `iv` follows from its seed, with the stored order drawn from a -//! generator started at the same seed (`StackCipherBuilder::json_order_seed`, +//! generator started at the same seed (`StackCipherBuilder::json_seed`, //! behind the `test-support` feature, which this file needs), so every one //! of them is the same in every run. The ciphertexts carry a fresh nonce //! each and are not pinned. In production the order is drawn from the @@ -192,7 +192,7 @@ async fn the_derivations_are_pinned_under_the_deterministic_source() { for root in ["sealed_root", "open_root"] { assert_eq!( pinned[root]["order"], actual[root]["order"], - "{root}: the stored order the seeded generator draws moved: a change to how the order is drawn (json::order) or to the generator, not a stored-data break" + "{root}: the stored order the seeded generator draws moved: a change to how the order is drawn (json::shuffle) or to the generator, not a stored-data break" ); } assert_eq!(pinned, actual, "{DRIFT}"); From b1e3db765e410ec30dfb335805c9e416493789a5 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 17:59:34 +1100 Subject: [PATCH 12/22] refactor(stack-encrypt): hold the JSON index's owned documents in Protected The index had two ways to wipe a serde_json value it owned: `PlainJson`, the crate's `Zeroize` newtype that `Protected` wipes on drop, and a second `Wiped` wrapper with its own `wipe_value`, used by sealing, by the query checks and by the owned-source release. The second is gone: every document this crate owns is a `Protected` and is wiped by vitaminc when it drops. Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- packages/stack-encrypt/src/json/derive.rs | 59 +++---------------- packages/stack-encrypt/src/json/mod.rs | 3 +- packages/stack-encrypt/src/json/query.rs | 30 +++++----- .../stack-encrypt/src/target/operations.rs | 2 +- 4 files changed, 26 insertions(+), 68 deletions(-) diff --git a/packages/stack-encrypt/src/json/derive.rs b/packages/stack-encrypt/src/json/derive.rs index 185ddc6fd..f653340fb 100644 --- a/packages/stack-encrypt/src/json/derive.rs +++ b/packages/stack-encrypt/src/json/derive.rs @@ -20,7 +20,7 @@ use vitaminc_encrypt::{Aes256Cipher, Key as AesKey}; use vitaminc_hmac::HmacSha256Prf; use vitaminc_prf::{BlockVisitor, IntoPrfContext, PrfValue}; use vitaminc_protected::{AsProtectedRef, Controlled, Protected}; -use zeroize::{Zeroize, Zeroizing}; +use zeroize::Zeroizing; use super::cascade::{self, ChildrenDigest}; use super::path::{key_step_len, push_key, JsonPath, Segment, ARRAY_STEP}; @@ -810,15 +810,20 @@ pub(crate) fn seal_pending<'a, K: 'static>( return Pending::failed(cipher, error); } // The caller's document, owned here: wiped when this returns. - let value = Wiped(value); + let value = PlainJson::protect(value); let prf_context = piece.clone().into_prf_context().into_owned(); let aad = piece.into_aad().into_owned(); - let walked = match walk(&value, options, cipher.prf(), &prf_context) { + let walked = match walk( + value.risky_ref().value(), + options, + cipher.prf(), + &prf_context, + ) { Ok(walked) => walked, Err(error) => return Pending::failed(cipher, error.into()), }; let whole = if options.sealed_root { - match sealed_root_plaintext(&value) { + match sealed_root_plaintext(value.risky_ref().value()) { Ok(bytes) => Some(bytes), Err(error) => return Pending::failed(cipher, Error::Other(Box::new(error))), } @@ -841,39 +846,6 @@ pub(crate) fn seal_pending<'a, K: 'static>( ) } -/// A document this crate owns, its strings and key names wiped when it -/// drops. -pub(crate) struct Wiped(pub(crate) Value); - -impl std::ops::Deref for Wiped { - type Target = Value; - fn deref(&self) -> &Value { - &self.0 - } -} - -impl Drop for Wiped { - fn drop(&mut self) { - wipe_value(&mut self.0); - } -} - -/// Wipe every string and key name in `value`, leaving it empty: numbers, -/// booleans and nulls hold nothing outside the value itself. -pub(crate) fn wipe_value(value: &mut Value) { - match value { - Value::String(text) => text.zeroize(), - Value::Array(items) => items.iter_mut().for_each(wipe_value), - Value::Object(map) => { - for (mut key, mut child) in std::mem::take(map) { - wipe_value(&mut child); - key.zeroize(); - } - } - Value::Null | Value::Bool(_) | Value::Number(_) => {} - } -} - /// An entry's AAD: the domain, the keyset, the column's AAD, the key tag, /// the entry's selector, the slot `j` it is stored at, and the digest of its /// children's AEAD tags ([`cascade::children_digest`]). The last is what @@ -2239,19 +2211,6 @@ mod tests { } } - #[test] - fn wiping_a_value_empties_its_strings_and_keys() { - let mut value = json!({"key": ["text", 1, null, {"inner": "more"}], "n": 2.5}); - wipe_value(&mut value); - assert_eq!(value, json!({}), "every member taken and wiped"); - let mut text = json!("text"); - wipe_value(&mut text); - assert_eq!(text, json!("")); - let mut list = json!(["a", ["b"], true]); - wipe_value(&mut list); - assert_eq!(list, json!(["", [""], true])); - } - #[test] fn numbers_canonicalise_by_value() { let n = |text: &str| -> Number { text.parse().unwrap() }; diff --git a/packages/stack-encrypt/src/json/mod.rs b/packages/stack-encrypt/src/json/mod.rs index d17ffe839..de485641d 100644 --- a/packages/stack-encrypt/src/json/mod.rs +++ b/packages/stack-encrypt/src/json/mod.rs @@ -135,7 +135,8 @@ pub use query::{ AtPathTerms, JsonLeafQuery, JsonProjection, JsonQuery, JsonSelector, JsonSubtreeKey, }; -pub(crate) use derive::{seal_pending, terms_pending, wipe_value}; +pub(crate) use derive::{seal_pending, terms_pending}; +pub(crate) use plain::PlainJson; pub(crate) use shuffle::Randomness; /// A sixteen-byte tag: a selector or a token. diff --git a/packages/stack-encrypt/src/json/query.rs b/packages/stack-encrypt/src/json/query.rs index d63434092..30044fee3 100644 --- a/packages/stack-encrypt/src/json/query.rs +++ b/packages/stack-encrypt/src/json/query.rs @@ -14,6 +14,7 @@ use serde_json::Value; use super::derive::{JsonTerms, Selectors, TermNode}; use super::path::JsonPath; +use super::plain::PlainJson; use super::{base64, JsonDocument, JsonExtracted, JsonNode, JsonOptions, JsonSubtree, Tag}; use crate::op::{ Answers, AtPath, Between, ContainedBy, Contains, DefaultIndex, Eq, Gt, Gte, Input, @@ -22,6 +23,7 @@ use crate::op::{ use crate::sem::{RangeQuery, TermError}; use crate::target::{DeclaredContext, Index, Json, Owned, Pending}; use crate::{Error, KeysetCipher, Label, NonEmpty}; +use vitaminc_protected::{Controlled, Protected}; /// One selector pair of a projection. #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] @@ -706,7 +708,7 @@ where type Verifier = (); fn check(&self, (path, value): &(P, V)) -> Result<(), Error> { let _ = parse_path(path.as_ref())?; - wiped_after(value.clone().into(), scalar) + scalar(©_of(value)) } fn verifier(&self, _: &(P, V)) {} fn query((): (), terms: AtPathTerms, _: QueryToken) -> JsonQuery { @@ -728,19 +730,14 @@ where type Via = Json; } -/// Whether `value` is a scalar, which equality at a path compares. -/// `check` over `value`, a copy this crate made of the caller's plaintext, -/// which is wiped afterwards rather than dropped. -fn wiped_after( - mut value: Value, - check: impl FnOnce(&Value) -> Result<(), Error>, -) -> Result<(), Error> { - let checked = check(&value); - super::wipe_value(&mut value); - checked +/// The copy of the caller's plaintext a check reads: wiped when it drops. +fn copy_of + Clone>(value: &V) -> Protected { + PlainJson::protect(value.clone().into()) } -fn scalar(value: &Value) -> Result<(), Error> { +/// Whether `value` is a scalar, which equality at a path compares. +fn scalar(value: &Protected) -> Result<(), Error> { + let value = value.risky_ref().value(); match value { Value::Object(_) | Value::Array(_) => Err(TermError::JsonValueNotScalar.into()), _ => Ok(()), @@ -748,7 +745,8 @@ fn scalar(value: &Value) -> Result<(), Error> { } /// Whether `value` is a leaf the index orders: a number or a string. -fn ordered(value: &Value) -> Result<(), Error> { +fn ordered(value: &Protected) -> Result<(), Error> { + let value = value.risky_ref().value(); match value { Value::Number(_) | Value::String(_) => Ok(()), _ => Err(TermError::JsonLeafNotOrdered.into()), @@ -787,7 +785,7 @@ macro_rules! orderings { fn check(&self, (path, value): &(P, V)) -> Result<(), Error> { let path = parse_path(path.as_ref())?; orders(self, &path)?; - wiped_after(value.clone().into(), ordered) + ordered(©_of(value)) } fn verifier(&self, _: &(P, V)) {} fn query((): (), terms: AtPathTerms, _: QueryToken) -> JsonLeafQuery { @@ -822,8 +820,8 @@ where fn check(&self, (path, (lo, hi)): &(P, (V, V))) -> Result<(), Error> { let path = parse_path(path.as_ref())?; orders(self, &path)?; - wiped_after(lo.clone().into(), ordered)?; - wiped_after(hi.clone().into(), ordered) + ordered(©_of(lo))?; + ordered(©_of(hi)) } fn verifier(&self, _: &(P, (V, V))) {} fn query( diff --git a/packages/stack-encrypt/src/target/operations.rs b/packages/stack-encrypt/src/target/operations.rs index b74fec3af..f546dec3f 100644 --- a/packages/stack-encrypt/src/target/operations.rs +++ b/packages/stack-encrypt/src/target/operations.rs @@ -582,7 +582,7 @@ pub fn json_terms<'s, K: 'static, M: SourceMode<'s, serde_json::Value>>( }; // The terms are derived by now; a document this description // owns (a query's needle) is wiped rather than dropped. - M::release(source, |mut value| crate::json::wipe_value(&mut value)); + M::release(source, |value| drop(crate::json::PlainJson::protect(value))); pending }), } From 93ccdb9eb8a53bdf63eb96c90b52fc4b0cac19e7 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 18:04:51 +1100 Subject: [PATCH 13/22] refactor(stack-encrypt): name the JSON index's selector set and compare it in constant time `JsonProjection` did not say what it held. It is the selector pairs of a document's nodes grouped by array element, what a stored document carries as `q` and what a containment query sends, so it is `JsonSelectorSet`, and the accessors on `JsonDocument` and `JsonQuery` are `selector_set`. It holds no plaintext, only keyed hashes the database stores, but they are derived from the document's keys and values, so it prints through `OpaqueDebug`, and its set equality runs through vitaminc's `TimingSafeEq` rather than a short-circuiting comparison. `JsonQuery` prints opaquely too. The query a containment sends was a "needle" in the code; the identifiers now say what each thing is: the terms carry their `query_selectors`, and a path builds the document `holding` a value. Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- docs/fuzzing.md | 2 +- docs/plans/2026-10-17-json-tree-index.md | 2 +- .../eql/tests/encryption/tests/json_tree.rs | 2 +- .../fuzz/fuzz_targets/json_wire_decode.rs | 4 +- packages/stack-encrypt/src/cipher.rs | 8 +- packages/stack-encrypt/src/json/derive.rs | 57 ++-- packages/stack-encrypt/src/json/mod.rs | 41 +-- packages/stack-encrypt/src/json/path.rs | 18 +- packages/stack-encrypt/src/json/query.rs | 267 ++++++++++-------- .../stack-encrypt/src/target/operations.rs | 2 +- packages/stack-encrypt/tests/common/mod.rs | 2 +- packages/stack-encrypt/tests/json_index.rs | 30 +- .../stack-encrypt/tests/json_tree_vectors.rs | 2 +- 13 files changed, 234 insertions(+), 203 deletions(-) diff --git a/docs/fuzzing.md b/docs/fuzzing.md index ca130747b..86c212c3d 100644 --- a/docs/fuzzing.md +++ b/docs/fuzzing.md @@ -28,7 +28,7 @@ Current targets: | `fuzz:term-decode` | `stack-encrypt` | `term_decode` | the SEM term decoders (`EqualityTerm`, `MatchTerms`, `OreTerm`, `OpeTerm` `from_bytes`) | | `fuzz:check-record` | `stack-encrypt` | `check_record` | `dynamic::record::check_record` over an `Arbitrary`-derived plan and tree, against a model of the record rules (structure-aware) | | `fuzz:plan-build` | `stack-encrypt` | `plan_build` | `plan::FieldsBuilder::build` over `Arbitrary`-derived untrusted names (context, fields, identities); a built field's label must render and parse back, and passthrough names are never refused (structure-aware) | -| `fuzz:json-wire` | `stack-encrypt` | `json_wire_decode` | the JSON index's wire decoders (`JsonDocument`, `JsonNode`, `JsonProjection`, `JsonLeafQuery` from JSON text; `JsonHeader::from_bytes`), over base64 tags and base85 ciphertexts; an accepted document or node must round-trip through its wire form | +| `fuzz:json-wire` | `stack-encrypt` | `json_wire_decode` | the JSON index's wire decoders (`JsonDocument`, `JsonNode`, `JsonSelectorSet`, `JsonLeafQuery` from JSON text; `JsonHeader::from_bytes`), over base64 tags and base85 ciphertexts; an accepted document or node must round-trip through its wire form | Each target is a few lines — `libfuzzer-sys` hands a `&str` (or `&[u8]` for the byte decoders) to the parser via the `arbitrary` crate: diff --git a/docs/plans/2026-10-17-json-tree-index.md b/docs/plans/2026-10-17-json-tree-index.md index e25a1d552..61a6139f1 100644 --- a/docs/plans/2026-10-17-json-tree-index.md +++ b/docs/plans/2026-10-17-json-tree-index.md @@ -712,7 +712,7 @@ The stored order was the last disclosure of the format that no option changed: e 2. **A random stored order is the format.** Not an option, a flag or a switch. The root stays at slot 0, where the sealed-root fallback (`sv -> 0`) finds it; every other entry is placed by a uniformly random permutation of the slots 1…n, and `q`'s pair lists and element groups are shuffled the same way. The permutation is drawn per document at sealing time and thrown away: it is derived from nothing, keyed or not, because nobody needs to recompute it, and a permutation derived from anything stored (the `iv`) would be computable by the database. The generator is vitaminc's `SafeRand::from_entropy` with `next_below` (Lemire's reduction, bias at most n / 2⁶⁴) driving Fisher–Yates (`packages/stack-encrypt/src/json/order.rs`); it is not part of the format, since nothing stored depends on it. For fixtures, `StackCipherBuilder::json_order_seed` (behind `test-support`) draws every document's order from one stream started at a seed, through `SafeRand::from_controlled_seed`, and the deterministic test cipher sets it: `tests/fixtures/json_tree_vectors.json` now pins, per root option, the entries in their stored order, the path stored at each slot, the selector multiset, `q` as sets, and the tokens. The tags and tokens did not move. - What stays visible at rest, and no order hides: the multiset of selectors (paths, node counts, array lengths, whether two rows have the same paths in the same numbers), `q`'s grouping by array element, which entries are heads under the sealed root (the head count, no longer the size of each head's subtree), each ciphertext's length, and the ordering terms at ordered paths. The leakage page drops its stored-order row and says this; the executable profile in `tests/json_leakage` models the new order, reading every world in an order an observer can rebuild without the stored one (`canonical.rs`). A containment or equality needle is shuffled the same way, with a fresh permutation per query drawn from the same `OrderSource` (`JsonQuery::of_terms` and `at_path`; the seed travels in the query's `JsonTerms`), so the query-time profile matches the at-rest one: in the walk's order a needle would show which of two sibling keys sorts first. `jsonb @>` is set containment over arrays, so the order changes no answer, and `JsonProjection` (and so `JsonQuery`) compares as sets. Added at the owner's request after review, in the same PR. + What stays visible at rest, and no order hides: the multiset of selectors (paths, node counts, array lengths, whether two rows have the same paths in the same numbers), `q`'s grouping by array element, which entries are heads under the sealed root (the head count, no longer the size of each head's subtree), each ciphertext's length, and the ordering terms at ordered paths. The leakage page drops its stored-order row and says this; the executable profile in `tests/json_leakage` models the new order, reading every world in an order an observer can rebuild without the stored one (`canonical.rs`). A containment or equality needle is shuffled the same way, with a fresh permutation per query drawn from the same `OrderSource` (`JsonQuery::of_terms` and `at_path`; the seed travels in the query's `JsonTerms`), so the query-time profile matches the at-rest one: in the walk's order a needle would show which of two sibling keys sorts first. `jsonb @>` is set containment over arrays, so the order changes no answer, and `JsonSelectorSet` (and so `JsonQuery`) compares as sets. Added at the owner's request after review, in the same PR. 3. **First match at a wildcard path is unspecified (owner, 2026-10-26).** `json_tree_path_query_first`, and so `->`, `->>` and an ordering on `eql_v4.ord_term(col -> sel)`, returns the first entry in `sv` carrying a selector. At a wildcard path (every element under `$.a[*]`, at any depth) that is now an unspecified element, stable for a stored row but not element 0. Ordering or comparing over the elements of a wildcard path is written over all the matches (`json_tree_path_query`, `EXISTS` or `ORDER BY` over their ordering terms), and element order within a document is recovered after decryption from the labels, as it already was. No SQL changes. Two variants were considered and dropped by the owner: a fix-up after the shuffle that kept each wildcard selector's document-first entry first (it would have revealed, per wildcard selector, which entry is the document-first one), and writing the heads' digest into the sealed root's plaintext so that a client reading the root with its heads could verify them. The sealed root commits to nothing beneath it. diff --git a/packages/eql/tests/encryption/tests/json_tree.rs b/packages/eql/tests/encryption/tests/json_tree.rs index 6ae1431a9..c8ab49b69 100644 --- a/packages/eql/tests/encryption/tests/json_tree.rs +++ b/packages/eql/tests/encryption/tests/json_tree.rs @@ -232,7 +232,7 @@ async fn the_query_operands_are_the_engines_terms_in_the_sql_contract() { b.sort_unstable(); assert_eq!(a, b, "the same selectors"); assert_eq!( - as_set(&serde_json::to_value(engine.projection()).unwrap()), + as_set(&serde_json::to_value(engine.selector_set()).unwrap()), as_set(&serde_json::to_value(&typed.q).unwrap()), "and the same projection" ); diff --git a/packages/stack-encrypt/fuzz/fuzz_targets/json_wire_decode.rs b/packages/stack-encrypt/fuzz/fuzz_targets/json_wire_decode.rs index ae4fbd1bc..24370c02c 100644 --- a/packages/stack-encrypt/fuzz/fuzz_targets/json_wire_decode.rs +++ b/packages/stack-encrypt/fuzz/fuzz_targets/json_wire_decode.rs @@ -1,7 +1,7 @@ #![no_main] use libfuzzer_sys::fuzz_target; -use stack_encrypt::json::{JsonDocument, JsonHeader, JsonLeafQuery, JsonNode, JsonProjection}; +use stack_encrypt::json::{JsonDocument, JsonHeader, JsonLeafQuery, JsonNode, JsonSelectorSet}; // Fuzz the JSON index's wire decoders. A stored `json_tree` document, and // every entry `->` or a walk returns, come back from the database through @@ -29,6 +29,6 @@ fuzz_target!(|bytes: &[u8]| { let back: JsonNode = serde_json::from_value(wire).expect("its wire form reads back"); assert_eq!(back, node, "JsonNode wire form is not lossless"); } - let _ = serde_json::from_str::(text); + let _ = serde_json::from_str::(text); let _ = serde_json::from_str::(text); }); diff --git a/packages/stack-encrypt/src/cipher.rs b/packages/stack-encrypt/src/cipher.rs index dc6473c5b..7b5f281f2 100644 --- a/packages/stack-encrypt/src/cipher.rs +++ b/packages/stack-encrypt/src/cipher.rs @@ -532,8 +532,8 @@ pub struct StackCipher { /// first out. See [`keyset`](Self::keyset). keysets: Mutex, /// Where the JSON index draws its permutations from (a document's stored - /// order, a query's needle order): fresh entropy per document or query, - /// outside tests. + /// order, the order a query's selectors are sent in): fresh entropy per + /// document or query, outside tests. json_randomness: crate::json::Randomness, /// The backend's [`IndexKeySource::load_index_key`], captured at /// [`init`](StackCipherBuilder::init), where `K` is known to implement @@ -897,8 +897,8 @@ impl StackCipherBuilder { } /// Test support: draw every permutation the JSON index needs (the - /// stored order of each document it seals, the order of each query's - /// needle) from one generator started at `seed`, + /// stored order of each document it seals, the order each query's + /// selectors are sent in) from one generator started at `seed`, /// so that a run sealing the same documents in the same sequence draws /// the same permutations and a fixture can pin them. Paired with the /// deterministic key source (`kms::DeterministicSource`). diff --git a/packages/stack-encrypt/src/json/derive.rs b/packages/stack-encrypt/src/json/derive.rs index f653340fb..f8aed4525 100644 --- a/packages/stack-encrypt/src/json/derive.rs +++ b/packages/stack-encrypt/src/json/derive.rs @@ -28,7 +28,7 @@ use super::plain::{ insert_member, protected_array, protected_object, push_element, read_protected, take_protected, PlainJson, }; -use super::query::{group_of, JsonProjection, JsonSelector, JsonSubtreeKey, Read}; +use super::query::{group_of, JsonSelector, JsonSelectorSet, JsonSubtreeKey, Read}; use super::shuffle; use super::{JsonDocument, JsonEntry, JsonExtracted, JsonHeader, JsonOptions, JsonSubtree, Tag}; use crate::keyset::KeysetState; @@ -215,14 +215,14 @@ pub(crate) struct TermNode { pub struct JsonTerms { nodes: Vec, /// For a document whose root is a scalar: the selector pair the same - /// scalar has as an element of a root array, so a containment needle + /// scalar has as an element of a root array, so a containment query /// can match a stored `["a"]` as `jsonb @>` does. alt: Option<(Tag, Tag)>, - /// The projection a containment query built from these terms sends, + /// The selector set a containment query built from these terms sends, /// its pair lists and element groups in a random order drawn when the /// terms were derived for a query ([`terms_pending`]). `None` for the /// terms a document is sealed from, which no query is built from. - needle: Option, + query_selectors: Option, } /// The key names and the tokens are what the terms must not leak, so the @@ -265,14 +265,15 @@ impl JsonTerms { self.alt } - /// The projection a containment query sends: the one drawn when the + /// The selector set a containment query sends: the one drawn when the /// terms were derived for a query, or, for terms no query was derived - /// for, the nodes' projection in the walk's order. - pub(crate) fn into_needle(self) -> JsonProjection { - self.needle.unwrap_or_else(|| group_of(&self.nodes, None)) + /// for, the nodes' set in the walk's order. + pub(crate) fn into_query_selectors(self) -> JsonSelectorSet { + self.query_selectors + .unwrap_or_else(|| group_of(&self.nodes, None)) } - /// The last node of the walk: in a needle built from one path, the + /// The last node of the walk: in a document built from one path, the /// leaf the path names. pub(crate) fn leaf(&self) -> Option<&TermNode> { self.nodes.last() @@ -614,7 +615,7 @@ fn walk( terms: JsonTerms { nodes: walker.nodes, alt, - needle: None, + query_selectors: None, }, plain: walker.plain, }) @@ -768,9 +769,9 @@ fn shuffled_ordinals<'m>( // ============================================================================= /// The one-way terms of `value` under `context`, as the index's operation -/// derives them for a query: no key is requested. The projection a +/// derives them for a query: no key is requested. The selector set a /// containment query sends is drawn here, in a random order of its own -/// ([`shuffle`]), so a needle shows no more of the order of its keys than a +/// ([`shuffle`]), so a query shows no more of the order of its keys than a /// stored `q` does; `jsonb @>` reads both as sets, so the order changes no /// answer. pub(crate) fn terms_pending<'a, K>( @@ -784,10 +785,10 @@ pub(crate) fn terms_pending<'a, K>( .map_err(Error::from) .and_then(|walked| { let mut rng = cipher.cipher().json_randomness().generator()?; - let mut needle = group_of(&walked.terms.nodes, None); - needle.shuffle(&mut rng); + let mut selectors = group_of(&walked.terms.nodes, None); + selectors.shuffle(&mut rng); Ok(JsonTerms { - needle: Some(needle), + query_selectors: Some(selectors), ..walked.terms }) }); @@ -1025,12 +1026,12 @@ fn seal( } // `slots` is a permutation, so every slot was filled once. let entries: Vec = entries.into_iter().flatten().collect(); - let mut projection = group_of(nodes, None); - projection.shuffle(&mut rng); + let mut selectors = group_of(nodes, None); + selectors.shuffle(&mut rng); Ok(JsonDocument::new( JsonHeader::new(keyset_id, iv, tag, sealed_root), entries, - projection, + selectors, )) } @@ -2662,28 +2663,28 @@ mod tests { } proptest! { - /// A needle sent in a shuffled order matches exactly the stored - /// projections the needle in the walk's order matched, stored + /// A query sent in a shuffled order matches exactly the stored + /// selector sets the query in the walk's order matched, stored /// in either order: `jsonb @>` is set containment over arrays. #[test] - fn a_shuffled_needle_matches_what_the_unshuffled_one_did( + fn a_shuffled_query_matches_what_the_unshuffled_one_did( stored in arbitrary_json(), - needle in arbitrary_json(), + asked in arbitrary_json(), seed in any::<[u8; 32]>(), ) { let options = JsonOptions::default(); - let project = |value: &Value| super::super::super::query::group_of(terms(value, &options).nodes(), None); + let selectors = |value: &Value| group_of(terms(value, &options).nodes(), None); let mut rng = SafeRand::from_controlled_seed(Protected::new(seed)); - let walked = project(&needle); + let walked = selectors(&asked); let mut shuffled = walked.clone(); shuffled.shuffle(&mut rng); - let mut stored_shuffled = project(&stored); + let mut stored_shuffled = selectors(&stored); stored_shuffled.shuffle(&mut rng); - prop_assert_eq!(&shuffled, &walked, "one needle, as sets"); - for target in [project(&stored), stored_shuffled] { + prop_assert_eq!(&shuffled, &walked, "one query, as sets"); + for target in [selectors(&stored), stored_shuffled] { prop_assert_eq!(target.contains(&shuffled), target.contains(&walked)); } - prop_assert!(project(&needle).contains(&shuffled), "a needle contains itself in any order"); + prop_assert!(selectors(&asked).contains(&shuffled), "a query's set contains itself in any order"); } } diff --git a/packages/stack-encrypt/src/json/mod.rs b/packages/stack-encrypt/src/json/mod.rs index de485641d..805c29f78 100644 --- a/packages/stack-encrypt/src/json/mod.rs +++ b/packages/stack-encrypt/src/json/mod.rs @@ -34,11 +34,12 @@ //! same OPE term the scalar [`Ope`](crate::target::Ope) index derives, //! keyed by the column. No path is ordered by default. //! -//! Beside the entries the document stores its containment **projection** -//! `q`: one `(s, vs)` pair per node, `vs` being a PRF of the node's kind -//! and, for a scalar, its canonical value, bound to `s`, nested by array -//! element so that PostgreSQL's own `jsonb @>` over two projections is -//! exact containment. The value selectors live there and nowhere else. +//! Beside the entries the document stores its **selector set** `q` (the +//! design's containment projection, [`JsonSelectorSet`]): one `(s, vs)` +//! pair per node, `vs` being a PRF of the node's kind and, for a scalar, +//! its canonical value, bound to `s`, nested by array element so that +//! PostgreSQL's own `jsonb @>` over two sets is exact containment. The +//! value selectors live there and nowhere else. //! //! By default the root is **sealed** ([`JsonOptions::sealed_root`]): the root //! entry's ciphertext is the whole document and the top-level fields are @@ -132,7 +133,7 @@ mod shuffle; pub use derive::JsonTerms; pub use path::{JsonPath, JsonPathError, Segment}; pub use query::{ - AtPathTerms, JsonLeafQuery, JsonProjection, JsonQuery, JsonSelector, JsonSubtreeKey, + AtPathTerms, JsonLeafQuery, JsonQuery, JsonSelector, JsonSelectorSet, JsonSubtreeKey, }; pub(crate) use derive::{seal_pending, terms_pending}; @@ -433,14 +434,14 @@ impl JsonEntry { /// /// Serializes to the wire form the EQL domain stores: `{"h": "…", "sv": /// [{"s": "…", "c": "…", …}, …], "q": {"sv": [{"s", "vs"}, …], "el": -/// […]}}`, where `q` is the document's containment -/// [projection](Self::projection); see the [module docs](self#byte-encodings). +/// […]}}`, where `q` is the document's [selector set](Self::selector_set); +/// see the [module docs](self#byte-encodings). #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(try_from = "DocumentWire", into = "DocumentWire")] pub struct JsonDocument { header: JsonHeader, entries: Vec, - projection: JsonProjection, + selectors: JsonSelectorSet, } impl JsonDocument { @@ -457,13 +458,13 @@ impl JsonDocument { &self.entries } - /// The document's containment projection: what a [`JsonQuery`] is - /// matched against, stored as `q`. Computed from the plaintext when the + /// The document's selector set: what a [`JsonQuery`] is matched + /// against, stored as `q`. Computed from the plaintext when the /// document is sealed and read back as stored: like every search field /// it is unauthenticated (only `c` is), and a tampered `q` changes /// which rows a containment query returns and nothing else. - pub fn projection(&self) -> &JsonProjection { - &self.projection + pub fn selector_set(&self) -> &JsonSelectorSet { + &self.selectors } /// The entry whose self tag is `token`'s, as the server finds the node @@ -581,12 +582,12 @@ impl JsonDocument { pub(crate) fn new( header: JsonHeader, entries: Vec, - projection: JsonProjection, + selectors: JsonSelectorSet, ) -> Self { Self { header, entries, - projection, + selectors, } } } @@ -806,7 +807,7 @@ fn unbase85(text: &str) -> Result, WireError> { #[error("a JSON index wire field does not decode: {0}")] pub struct WireError(&'static str); -/// `h`, `sv` and `q`, as stored. All three are required: the projection +/// `h`, `sv` and `q`, as stored. All three are required: the selector set /// `q` is the only place the value selectors live. #[derive(Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields)] @@ -862,7 +863,7 @@ impl From for DocumentWire { Self { h: base64(&doc.header.to_bytes()), sv: doc.entries.iter().map(EntryWire::from).collect(), - q: query::GroupWire::from(&doc.projection), + q: query::GroupWire::from(&doc.selectors), } } } @@ -877,11 +878,11 @@ impl TryFrom for JsonDocument { w.sv.into_iter() .map(JsonEntry::try_from) .collect::>()?; - let projection = JsonProjection::try_from(w.q)?; + let selectors = JsonSelectorSet::try_from(w.q)?; Ok(Self { header, entries, - projection, + selectors, }) } } @@ -1039,7 +1040,7 @@ mod tests { let chain = JsonDocument::new( JsonHeader::new(Uuid::from_u128(7), iv, vec![], true), entries, - JsonProjection::default(), + JsonSelectorSet::default(), ); let walked = chain.subtree(&[token]); assert_eq!( diff --git a/packages/stack-encrypt/src/json/path.rs b/packages/stack-encrypt/src/json/path.rs index e1eb8fa31..95b327e89 100644 --- a/packages/stack-encrypt/src/json/path.rs +++ b/packages/stack-encrypt/src/json/path.rs @@ -330,9 +330,9 @@ impl JsonPath { /// The document that holds `value` at this path and nothing else: /// `$.a[*].x` with `1` is `{"a": [{"x": 1}]}`. A positional array step - /// places the value at element zero, since the needle's selectors do not - /// tell positions apart. - pub(crate) fn needle(&self, value: serde_json::Value) -> serde_json::Value { + /// places the value at element zero, since the selectors of a query do + /// not tell positions apart. + pub(crate) fn holding(&self, value: serde_json::Value) -> serde_json::Value { self.segments .iter() .rev() @@ -609,13 +609,13 @@ mod tests { } #[test] - fn a_needle_holds_the_value_at_the_path() { - let needle = JsonPath::parse("$.a[*].x") + fn the_document_holding_a_value_puts_it_at_the_path() { + let holding = JsonPath::parse("$.a[*].x") .unwrap() - .needle(serde_json::json!(1)); - assert_eq!(needle, serde_json::json!({"a": [{"x": 1}]})); - let needle = JsonPath::root().needle(serde_json::json!("v")); - assert_eq!(needle, serde_json::json!("v")); + .holding(serde_json::json!(1)); + assert_eq!(holding, serde_json::json!({"a": [{"x": 1}]})); + let holding = JsonPath::root().holding(serde_json::json!("v")); + assert_eq!(holding, serde_json::json!("v")); } #[test] diff --git a/packages/stack-encrypt/src/json/query.rs b/packages/stack-encrypt/src/json/query.rs index 30044fee3..2b4278ba9 100644 --- a/packages/stack-encrypt/src/json/query.rs +++ b/packages/stack-encrypt/src/json/query.rs @@ -1,12 +1,12 @@ //! The query terms of the JSON index, and how the operators reach it. //! -//! A containment needle is the **element-nested projection** of the -//! needle's terms: the selector pairs `{s, vs}` of every node, grouped by -//! the array element they sit in. A stored document carries its own -//! projection, built the same way, as `q`, which the server reads, and -//! native `jsonb @>` over the two projections is exact containment: each needle element has to fit inside -//! one stored element, which is PostgreSQL's own rule for arrays. The -//! design's "The preferred form" records why no re-check is needed. +//! A containment query sends the [`JsonSelectorSet`] of the document it +//! asks about: the selector pairs `{s, vs}` of every node, grouped by the +//! array element they sit in. A stored document carries its own, built the +//! same way, as `q`, which the server reads, and native `jsonb @>` over the +//! two sets is exact containment: each element of the query's document has +//! to fit inside one stored element, which is PostgreSQL's own rule for +//! arrays. use std::fmt; use serde::{Deserialize, Serialize}; @@ -23,41 +23,66 @@ use crate::op::{ use crate::sem::{RangeQuery, TermError}; use crate::target::{DeclaredContext, Index, Json, Owned, Pending}; use crate::{Error, KeysetCipher, Label, NonEmpty}; -use vitaminc_protected::{Controlled, Protected}; +use vitaminc_protected::{Choice, Controlled, OpaqueDebug, Protected, TimingSafeEq}; -/// One selector pair of a projection. -#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +/// One node's selector pair: its path selector `s` and its value selector +/// `vs`. Ordered, so a set can be sorted into one spelling. +#[derive(Clone, Copy, TimingSafeEq, PartialOrd, Ord)] struct Pair16 { s: Tag, vs: Tag, } -/// The element-nested projection of a document: the selector pairs that -/// sit outside any array element, and one nested projection per array -/// element that is a container. What a stored document carries as `q`, -/// which the server reads (`json_tree_query`), and what a containment -/// needle is. +/// The selector pairs `{s, vs}` of a document's nodes, grouped by array +/// element: the pairs of the nodes outside any array element, and one +/// nested set per array element that is a container. A stored document +/// carries its own as `q`, which the server reads (`json_tree_query`), and +/// a containment query sends the set of the document it asks about. /// -/// Equality is as sets, which is how `jsonb @>` reads a projection: two -/// projections are equal when they hold the same pairs in the same groups, -/// whatever order their lists are in. A stored `q` is shuffled, so two -/// seals of one document carry equal projections in different orders. -#[derive(Clone, Debug, Default)] -pub struct JsonProjection { +/// It holds no plaintext: a selector is a keyed hash of a path, and a value +/// selector of a path and a value, which is what the database stores. It +/// prints opaquely all the same, since the pairs are derived from the +/// document's keys and values. +/// +/// Equality is as sets, which is how `jsonb @>` reads one: two sets are +/// equal when they hold the same pairs in the same groups, whatever order +/// their lists are in. A stored `q` is shuffled, so two seals of one +/// document carry equal sets in different orders. The comparison is +/// constant-time in the pairs ([`TimingSafeEq`]); the sort that puts both +/// sides in one spelling first depends on the order of the pairs, which +/// the database holds. +#[derive(Clone, Default, OpaqueDebug)] +pub struct JsonSelectorSet { sv: Vec, - el: Vec, + el: Vec, } -impl PartialEq for JsonProjection { +impl TimingSafeEq for JsonSelectorSet { + fn ts_eq(&self, other: &Self) -> Choice { + let (a, b) = (self.canonical(), other.canonical()); + let mut equal = Choice::from(u8::from( + a.sv.len() == b.sv.len() && a.el.len() == b.el.len(), + )); + for (x, y) in a.sv.iter().zip(&b.sv) { + equal &= x.ts_eq(y); + } + for (x, y) in a.el.iter().zip(&b.el) { + equal &= x.ts_eq(y); + } + equal + } +} + +impl PartialEq for JsonSelectorSet { fn eq(&self, other: &Self) -> bool { - Self::cmp_canonical(&self.canonical(), &other.canonical()).is_eq() + bool::from(self.ts_eq(other)) } } -impl std::cmp::Eq for JsonProjection {} +impl std::cmp::Eq for JsonSelectorSet {} -impl JsonProjection { - /// This projection with every list sorted, recursively: one spelling per - /// set of pairs and groups. +impl JsonSelectorSet { + /// This set with every list sorted, recursively: one spelling per set + /// of pairs and groups. fn canonical(&self) -> Self { let mut sv = self.sv.clone(); sv.sort_unstable(); @@ -66,8 +91,9 @@ impl JsonProjection { Self { sv, el } } - /// The order of two projections already in [`canonical`](Self::canonical) - /// form: by their pairs, then by their groups in turn. + /// The order of two sets already in [`canonical`](Self::canonical) + /// form: by their pairs, then by their groups in turn. A sort key, not a + /// comparison of secrets. fn cmp_canonical(a: &Self, b: &Self) -> std::cmp::Ordering { a.sv.cmp(&b.sv).then_with(|| { a.el.iter() @@ -91,14 +117,14 @@ impl JsonProjection { } } - /// `jsonb @>` over two projections: every needle pair is some pair of - /// this projection, and every needle element fits inside some element - /// of it, recursively. What the server computes natively over the two - /// projections; here so a caller can decide containment over documents + /// `jsonb @>` over two sets: every pair of `other` is some pair of this + /// set, and every element group of `other` fits inside some element + /// group of it, recursively. What the server computes natively over + /// the two sets; here so a caller can decide containment over documents /// it holds, with no database. - pub fn contains(&self, needle: &JsonProjection) -> bool { - needle.sv.iter().all(|pair| self.sv.contains(pair)) - && needle + pub fn contains(&self, other: &JsonSelectorSet) -> bool { + other.sv.iter().all(|pair| self.sv.contains(pair)) + && other .el .iter() .all(|group| self.el.iter().any(|mine| mine.contains(group))) @@ -106,67 +132,69 @@ impl JsonProjection { } /// The query term of [`Contains`], [`ContainedBy`] and `AtPath`: the -/// element-nested projection of the needle's selectors, and, for a needle -/// whose root is a scalar, the alternative projection that matches a +/// [`JsonSelectorSet`] of the document the query asks about, and, for a +/// document whose root is a scalar, the alternative set that matches a /// stored root array holding it. Serializes to the `query_json_tree` /// operand: `{"sv": [{"s": "…", "vs": "…"}, …], "el": [{"sv": …, "el": …}, /// …], "alt": {…}}`. /// -/// The projection is sent in a random order of its own, drawn when the -/// terms were derived ([`terms_pending`](super::derive::terms_pending)) as a -/// stored `q`'s order is drawn when its document is sealed: every pair list -/// and element-group list shuffled, recursively. `jsonb @>` is set -/// containment over arrays, so the order changes no answer, and a needle -/// shows no more of the order of its keys than a stored row does. Two -/// derivations of one value therefore serialize differently and compare -/// equal (equality is as sets). -#[derive(Clone, Debug, PartialEq, Eq)] +/// The set is sent in a random order of its own, drawn when the terms were +/// derived ([`terms_pending`](super::derive::terms_pending)) as a stored +/// `q`'s order is drawn when its document is sealed: every pair list and +/// element-group list shuffled, recursively. `jsonb @>` is set containment +/// over arrays, so the order changes no answer, and a query shows no more +/// of the order of its keys than a stored row does. Two derivations of one +/// value therefore serialize differently and compare equal (equality is as +/// sets). +#[derive(Clone, OpaqueDebug, PartialEq, Eq)] pub struct JsonQuery { - group: JsonProjection, - alt: Option, + selectors: JsonSelectorSet, + alt: Option, } impl JsonQuery { - /// The needle of an accessor: the projection of the document that holds - /// the value at the path, with no `alt`. The alternative matches a root - /// array that holds a root scalar, which is containment; at a path, - /// `("$", 1)` asks for the scalar `1`, not for an array holding it. + /// The query of an accessor: the selector set of the document that + /// holds the value at the path, with no `alt`. The alternative matches + /// a root array that holds a root scalar, which is containment; at a + /// path, `("$", 1)` asks for the scalar `1`, not for an array holding + /// it. pub(crate) fn at_path(terms: JsonTerms) -> Self { Self { - group: terms.into_needle(), + selectors: terms.into_query_selectors(), alt: None, } } pub(crate) fn of_terms(terms: JsonTerms) -> Self { - let alt = terms.alt().map(|(s, vs)| JsonProjection { + let alt = terms.alt().map(|(s, vs)| JsonSelectorSet { sv: vec![Pair16 { s, vs }], el: Vec::new(), }); Self { - group: terms.into_needle(), + selectors: terms.into_query_selectors(), alt, } } - /// The needle's projection: what `<@` compares a stored document's - /// projection against. - pub fn projection(&self) -> &JsonProjection { - &self.group + /// The query's selector set: what `<@` compares a stored document's + /// against. + pub fn selector_set(&self) -> &JsonSelectorSet { + &self.selectors } - /// Whether a stored document's projection contains this needle, as the - /// server decides `@>`: containment of the needle's projection, or of - /// its alternative for a root scalar against a root array. - pub fn matches(&self, stored: &JsonProjection) -> bool { - stored.contains(&self.group) || self.alt.as_ref().is_some_and(|alt| stored.contains(alt)) + /// Whether a stored document's selector set contains this query's, as + /// the server decides `@>`: containment of the query's set, or of its + /// alternative for a root scalar against a root array. + pub fn matches(&self, stored: &JsonSelectorSet) -> bool { + stored.contains(&self.selectors) + || self.alt.as_ref().is_some_and(|alt| stored.contains(alt)) } } -/// The group of the nodes that sit directly in `parent` (`None` is the top -/// level), with the container elements in it nested as groups. What a -/// needle is, and what a document writes to `q` when it is sealed. -pub(crate) fn group_of(nodes: &[TermNode], parent: Option) -> JsonProjection { +/// The selector set of the nodes that sit directly in `parent` (`None` is +/// the top level), with the container elements in it nested as groups. +/// What a query sends, and what a document writes to `q` when it is sealed. +pub(crate) fn group_of(nodes: &[TermNode], parent: Option) -> JsonSelectorSet { let belongs = |index: usize, node: &TermNode| { if node.container_element { Some(index) @@ -174,7 +202,7 @@ pub(crate) fn group_of(nodes: &[TermNode], parent: Option) -> JsonProject node.element } }; - JsonProjection { + JsonSelectorSet { sv: nodes .iter() .enumerate() @@ -196,7 +224,7 @@ struct PairWire { vs: String, } -/// One group, as a needle or a stored document's `q` spells it. +/// One group, as a query or a stored document's `q` spells it. #[derive(Clone, Serialize, Deserialize)] pub(crate) struct GroupWire { sv: Vec, @@ -211,8 +239,8 @@ struct QueryWire { alt: Option, } -impl From<&JsonProjection> for GroupWire { - fn from(group: &JsonProjection) -> Self { +impl From<&JsonSelectorSet> for GroupWire { + fn from(group: &JsonSelectorSet) -> Self { Self { sv: group .sv @@ -227,21 +255,21 @@ impl From<&JsonProjection> for GroupWire { } } -/// A projection serializes as a stored `q` and a needle do: `{"sv": [{"s", -/// "vs"}, …], "el": [group, …]}`. -impl Serialize for JsonProjection { +/// A selector set serializes as a stored `q` and a query do: `{"sv": +/// [{"s", "vs"}, …], "el": [group, …]}`. +impl Serialize for JsonSelectorSet { fn serialize(&self, s: S) -> Result { GroupWire::from(self).serialize(s) } } -impl<'de> Deserialize<'de> for JsonProjection { +impl<'de> Deserialize<'de> for JsonSelectorSet { fn deserialize>(d: D) -> Result { - JsonProjection::try_from(GroupWire::deserialize(d)?).map_err(serde::de::Error::custom) + JsonSelectorSet::try_from(GroupWire::deserialize(d)?).map_err(serde::de::Error::custom) } } -impl TryFrom for JsonProjection { +impl TryFrom for JsonSelectorSet { type Error = super::WireError; fn try_from(w: GroupWire) -> Result { let tag = |text: &str| { @@ -261,7 +289,7 @@ impl TryFrom for JsonProjection { el: w .el .into_iter() - .map(JsonProjection::try_from) + .map(JsonSelectorSet::try_from) .collect::>()?, }) } @@ -270,7 +298,7 @@ impl TryFrom for JsonProjection { impl Serialize for JsonQuery { fn serialize(&self, s: S) -> Result { QueryWire { - group: GroupWire::from(&self.group), + group: GroupWire::from(&self.selectors), alt: self.alt.as_ref().map(GroupWire::from), } .serialize(s) @@ -281,10 +309,10 @@ impl<'de> Deserialize<'de> for JsonQuery { fn deserialize>(d: D) -> Result { let wire = QueryWire::deserialize(d)?; Ok(Self { - group: JsonProjection::try_from(wire.group).map_err(serde::de::Error::custom)?, + selectors: JsonSelectorSet::try_from(wire.group).map_err(serde::de::Error::custom)?, alt: wire .alt - .map(JsonProjection::try_from) + .map(JsonSelectorSet::try_from) .transpose() .map_err(serde::de::Error::custom)?, }) @@ -542,7 +570,7 @@ impl Serialize for JsonSubtreeKey { } /// The terms of an accessor's input: the path it was asked at, and the -/// index's terms of the needle that holds the value there. What +/// index's terms of the document that holds the value there. What /// [`Input>`](Input) derives, and what the JSON index's /// [`Supports`] impls build an accessor's query from. #[derive(Clone, Debug)] @@ -557,7 +585,8 @@ impl AtPathTerms { &self.path } - /// The terms of the needle, or of both needles for a pair of bounds. + /// The terms of the document holding the value, or of both documents + /// for a pair of bounds. pub fn terms(&self) -> &T { &self.terms } @@ -576,8 +605,8 @@ fn parse_path(text: &str) -> Result { /// A path and a value: the input of `AtPath` and the single-bound /// orderings. The index is declared over the document, so the plaintext -/// is a [`Value`], and the term is derived over the needle that holds the -/// value at the path. +/// is a [`Value`], and the term is derived over the document that holds +/// the value at the path. impl Input> for (P, V) where P: AsRef + 'static, @@ -602,11 +631,11 @@ where Ok(path) => path, Err(error) => return Pending::failed(keyset, error), }; - let needle = path.needle(value.into()); + let holding = path.holding(value.into()); keyset .run( index.operation::().under(NonEmpty::from(label)), - needle, + holding, extend, ) .map(move |terms| AtPathTerms { path, terms }) @@ -614,7 +643,7 @@ where } /// A path and a pair of bounds: the input of `AtPath`. Both -/// needles derive under the one label, settled together. +/// documents' terms derive under the one label, settled together. impl Input> for (P, (V, V)) where P: AsRef + 'static, @@ -643,12 +672,12 @@ where index .operation::() .under(NonEmpty::from(label.clone())), - path.needle(lo.into()), + path.holding(lo.into()), extend.clone(), ); let hi = keyset.run( index.operation::().under(NonEmpty::from(label)), - path.needle(hi.into()), + path.holding(hi.into()), extend, ); lo.zip(hi).map(move |terms| AtPathTerms { path, terms }) @@ -667,8 +696,8 @@ where /// [`json_document`]: crate::target::json_document impl LabelledTarget for JsonDocument {} -/// Containment, both ways: the needle's projection, which the SQL compares -/// against the stored projection in the direction the operator says. +/// Containment, both ways: the query's selector set, which the SQL compares +/// against the stored set in the direction the operator says. macro_rules! containment { ($($op:ident),+ $(,)?) => {$( impl Supports<$op, Value> for Json { @@ -692,8 +721,8 @@ macro_rules! containment { } containment!(Contains, ContainedBy); -/// Equality at a path: the needle that holds the value at the path, -/// projected, with no `alt`. For a scalar, containment of that needle is +/// Equality at a path: the selector set of the document that holds the +/// value at the path, with no `alt`. For a scalar, containment of that set is /// equality of the value at the path. For an object or an array it would be /// containment (`{"x": 1}` matching `{"x": 1, "y": 2}`), so a container is /// refused with [`TermError::JsonValueNotScalar`] when the query is @@ -869,11 +898,11 @@ mod tests { } } - /// Projections compare as the sets `jsonb @>` reads: the order of their - /// lists does not count, their contents and their grouping do. + /// Selector sets compare as the sets `jsonb @>` reads: the order of + /// their lists does not count, their contents and their grouping do. #[test] - fn projections_are_equal_as_sets() { - let group = |sv: Vec, el: Vec| JsonProjection { sv, el }; + fn selector_sets_are_equal_as_sets() { + let group = |sv: Vec, el: Vec| JsonSelectorSet { sv, el }; let a = group( vec![pair(1), pair(2)], vec![ @@ -901,7 +930,7 @@ mod tests { assert_ne!(a, fewer, "a pair fewer"); let more_groups = group(a.sv.clone(), { let mut el = a.el.clone(); - el.push(JsonProjection::default()); + el.push(JsonSelectorSet::default()); el }); assert_ne!(a, more_groups, "a group more"); @@ -918,15 +947,15 @@ mod tests { use serde_json::json; #[test] - fn a_query_serializes_as_the_needle_operand() { + fn a_query_serializes_as_the_query_operand() { let pair = |b: u8| Pair16 { s: [b; 16], vs: [b + 1; 16], }; let query = JsonQuery { - group: JsonProjection { + selectors: JsonSelectorSet { sv: vec![pair(1)], - el: vec![JsonProjection { + el: vec![JsonSelectorSet { sv: vec![pair(3)], el: vec![], }], @@ -1011,13 +1040,13 @@ mod tests { } #[test] - fn a_projection_round_trips_through_its_wire_form() { - let projection = JsonProjection { + fn a_selector_set_round_trips_through_its_wire_form() { + let set = JsonSelectorSet { sv: vec![Pair16 { s: [1; 16], vs: [2; 16], }], - el: vec![JsonProjection { + el: vec![JsonSelectorSet { sv: vec![Pair16 { s: [3; 16], vs: [4; 16], @@ -1025,7 +1054,7 @@ mod tests { el: Vec::new(), }], }; - let wire = serde_json::to_value(&projection).unwrap(); + let wire = serde_json::to_value(&set).unwrap(); assert_eq!( wire, serde_json::json!({ @@ -1033,16 +1062,16 @@ mod tests { "el": [{"sv": [{"s": "AwMDAwMDAwMDAwMDAwMDAw==", "vs": "BAQEBAQEBAQEBAQEBAQEBA=="}], "el": []}], }) ); - let back: JsonProjection = serde_json::from_value(wire).unwrap(); - assert_eq!(back, projection); + let back: JsonSelectorSet = serde_json::from_value(wire).unwrap(); + assert_eq!(back, set); assert!( - serde_json::from_value::(serde_json::json!({"sv": [], "el": []})) + serde_json::from_value::(serde_json::json!({"sv": [], "el": []})) .unwrap() .el .is_empty() ); assert!( - serde_json::from_value::( + serde_json::from_value::( serde_json::json!({"sv": [{"s": "zz", "vs": "00"}], "el": []}) ) .is_err(), @@ -1056,28 +1085,28 @@ mod tests { s: [b; 16], vs: [b; 16], }; - let element = |pairs: &[u8]| JsonProjection { + let element = |pairs: &[u8]| JsonSelectorSet { sv: pairs.iter().map(|&b| pair(b)).collect(), el: vec![], }; - let stored = JsonProjection { + let stored = JsonSelectorSet { sv: vec![pair(0)], el: vec![element(&[1, 5]), element(&[7, 2])], }; // The issue's example: 1 and 2 sit in different elements. - assert!(!stored.contains(&JsonProjection { + assert!(!stored.contains(&JsonSelectorSet { sv: vec![], el: vec![element(&[1, 2])], })); - assert!(stored.contains(&JsonProjection { + assert!(stored.contains(&JsonSelectorSet { sv: vec![], el: vec![element(&[1, 5])], })); - assert!(stored.contains(&JsonProjection { + assert!(stored.contains(&JsonSelectorSet { sv: vec![pair(0)], el: vec![element(&[1]), element(&[2])], })); - assert!(!stored.contains(&JsonProjection { + assert!(!stored.contains(&JsonSelectorSet { sv: vec![pair(9)], el: vec![], })); diff --git a/packages/stack-encrypt/src/target/operations.rs b/packages/stack-encrypt/src/target/operations.rs index f546dec3f..07b608263 100644 --- a/packages/stack-encrypt/src/target/operations.rs +++ b/packages/stack-encrypt/src/target/operations.rs @@ -581,7 +581,7 @@ pub fn json_terms<'s, K: 'static, M: SourceMode<'s, serde_json::Value>>( Err(e) => Pending::failed(cipher, e), }; // The terms are derived by now; a document this description - // owns (a query's needle) is wiped rather than dropped. + // owns (the one a query asks about) is wiped rather than dropped. M::release(source, |value| drop(crate::json::PlainJson::protect(value))); pending }), diff --git a/packages/stack-encrypt/tests/common/mod.rs b/packages/stack-encrypt/tests/common/mod.rs index e761ea8a1..d0247a981 100644 --- a/packages/stack-encrypt/tests/common/mod.rs +++ b/packages/stack-encrypt/tests/common/mod.rs @@ -227,7 +227,7 @@ pub use stack_kms::DeterministicSource; /// A cipher over [`DeterministicSource`] with `seed`. With the crate's /// `test-support` feature, the JSON index's permutations (a document's -/// stored order, a needle's order) are drawn from a +/// stored order, the order a query's selectors are sent in) are drawn from a /// generator started at the same seed, so they repeat between runs too; /// without it they are random per document, as in production. pub async fn deterministic_cipher(seed: [u8; 32]) -> StackCipher { diff --git a/packages/stack-encrypt/tests/json_index.rs b/packages/stack-encrypt/tests/json_index.rs index 00ac923bb..0859cc2c7 100644 --- a/packages/stack-encrypt/tests/json_index.rs +++ b/packages/stack-encrypt/tests/json_index.rs @@ -455,7 +455,7 @@ mod given_the_fixture_document { "`q` round-trips as stored" ); assert_eq!( - serde_json::to_value(back.projection()).unwrap(), + serde_json::to_value(back.selector_set()).unwrap(), wire["q"], "the projection read back is the stored one" ); @@ -540,7 +540,7 @@ mod given_the_fixture_document { assert_eq!(tags.len(), 6, "every parent tag is distinct"); // The projection groups the two list elements: each is a nested group // holding its own pair and its `x`. - let q = serde_json::to_value(doc.projection()).unwrap(); + let q = serde_json::to_value(doc.selector_set()).unwrap(); assert_eq!( q["el"].as_array().unwrap().len(), 2, @@ -571,7 +571,7 @@ mod given_the_fixture_document { // elements' path, in their stored (shuffled) order. let element_vs = |doc: &JsonDocument| -> Vec { let s = base64(doc.entries()[1].selector()); - let q = serde_json::to_value(doc.projection()).unwrap(); + let q = serde_json::to_value(doc.selector_set()).unwrap(); q["sv"] .as_array() .unwrap() @@ -1775,8 +1775,8 @@ mod given_the_design_corpus { .await .unwrap(); let doc = cipher.encrypt(&value).using(&plan).await.unwrap(); - assert!(one.matches(doc.projection()) && two.matches(doc.projection())); - assert!(at.matches(doc.projection()), "an equality needle too"); + assert!(one.matches(doc.selector_set()) && two.matches(doc.selector_set())); + assert!(at.matches(doc.selector_set()), "an equality needle too"); } /// Every pair of the corpus, both ways round: `@>` over the projections @@ -2215,7 +2215,7 @@ mod given_generated_documents { case.stored ); prop_assert_eq!( - query.matches(doc.projection()), + query.matches(doc.selector_set()), pg_contains(&case.stored, &needle, true), "{} @> {}", &case.stored, @@ -2235,8 +2235,8 @@ mod given_generated_documents { .unwrap(); prop_assert_eq!(&again, &query, "one needle, as sets"); prop_assert_eq!( - again.matches(doc.projection()), - query.matches(doc.projection()), + again.matches(doc.selector_set()), + query.matches(doc.selector_set()), "{} @> {}, sent in another order", &case.stored, &needle @@ -2265,7 +2265,7 @@ mod given_generated_documents { excluded.set(excluded.get() + 1); } prop_assert_eq!( - query.projection().contains(doc.projection()), + query.selector_set().contains(doc.selector_set()), jsonb && !scalar_in_array(&case.stored, &needle), "{} <@ {}", &case.stored, @@ -2498,14 +2498,14 @@ mod given_equality_at_the_root { let scalar = cipher.encrypt(&json!(1)).using(&plan).await.unwrap(); let array = cipher.encrypt(&json!([1])).using(&plan).await.unwrap(); assert!( - at_root.matches(scalar.projection()), + at_root.matches(scalar.selector_set()), "(\"$\", 1) matches a stored 1" ); assert!( - !at_root.matches(array.projection()), + !at_root.matches(array.selector_set()), "(\"$\", 1) does not match a stored [1]" ); - assert!(contains.matches(array.projection()), "jsonb: [1] @> 1"); + assert!(contains.matches(array.selector_set()), "jsonb: [1] @> 1"); assert!( serde_json::to_value(&at_root).unwrap().get("alt").is_none(), "equality at the root carries no alternative" @@ -2520,7 +2520,7 @@ mod given_an_accessor { async fn matches_the_value_at_its_path() { let Scenario { cipher, plan } = scenario().await; let doc = cipher.encrypt(&fixture()).using(&plan).await.unwrap(); - let projection = doc.projection(); + let projection = doc.selector_set(); for (path, value, matches, what) in [ ("$.nested.list[*].x", json!(2), true, "an element's x is 2"), ("$.nested.list[*].x", json!(3), false, "no element's x is 3"), @@ -3094,7 +3094,7 @@ mod assertions { needle: &Value, ) { assert_eq!( - query.matches(doc.projection()), + query.matches(doc.selector_set()), pg_contains(stored, needle, true), "{stored} @> {needle}" ); @@ -3109,7 +3109,7 @@ mod assertions { needle: &Value, ) { assert_eq!( - query.projection().contains(doc.projection()), + query.selector_set().contains(doc.selector_set()), pg_contains(needle, stored, true) && !scalar_in_array(stored, needle), "{stored} <@ {needle}" ); diff --git a/packages/stack-encrypt/tests/json_tree_vectors.rs b/packages/stack-encrypt/tests/json_tree_vectors.rs index 58efc6be2..2a3c5bfb4 100644 --- a/packages/stack-encrypt/tests/json_tree_vectors.rs +++ b/packages/stack-encrypt/tests/json_tree_vectors.rs @@ -149,7 +149,7 @@ async fn vectors(options: &JsonOptions) -> Value { "entries": entries, "order": order, "selectors": selectors, - "q": as_set(&serde_json::to_value(doc.projection()).unwrap()), + "q": as_set(&serde_json::to_value(doc.selector_set()).unwrap()), "tokens": tokens, }) } From 731c4ef083130ed64a2b1b76ec12384993163851 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 18:10:04 +1100 Subject: [PATCH 14/22] chore: record stack-encrypt's rand dependency in the detached lockfiles The EQL workspace, the Go encrypt guest and the stack-encrypt fuzz crate resolve stack-encrypt by path, so each lockfile gains the one `rand` line; `cargo metadata --locked` passes in all three again. Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- languages/golang/encrypt/guest/Cargo.lock | 1 + packages/eql/Cargo.lock | 1 + packages/stack-encrypt/fuzz/Cargo.lock | 1 + 3 files changed, 3 insertions(+) diff --git a/languages/golang/encrypt/guest/Cargo.lock b/languages/golang/encrypt/guest/Cargo.lock index fa249ed7d..d161c425c 100644 --- a/languages/golang/encrypt/guest/Cargo.lock +++ b/languages/golang/encrypt/guest/Cargo.lock @@ -2166,6 +2166,7 @@ dependencies = [ "clww-ore", "miette", "ore-rs", + "rand 0.10.2", "serde", "serde_json", "serdect", diff --git a/packages/eql/Cargo.lock b/packages/eql/Cargo.lock index ba95bebae..702d8b1cc 100644 --- a/packages/eql/Cargo.lock +++ b/packages/eql/Cargo.lock @@ -4270,6 +4270,7 @@ dependencies = [ "clww-ore", "miette", "ore-rs 0.10.0", + "rand 0.10.1", "rust_decimal", "serde", "serde_json", diff --git a/packages/stack-encrypt/fuzz/Cargo.lock b/packages/stack-encrypt/fuzz/Cargo.lock index e9642b34c..b0d69f2e9 100644 --- a/packages/stack-encrypt/fuzz/Cargo.lock +++ b/packages/stack-encrypt/fuzz/Cargo.lock @@ -2082,6 +2082,7 @@ dependencies = [ "clww-ore", "miette", "ore-rs", + "rand 0.10.1", "serde", "serde_json", "serdect", From 2f0aa29025f5ed7f0a90acca32a38cd4c9555de4 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 18:10:04 +1100 Subject: [PATCH 15/22] refactor(stack-encrypt): name what the JSON index's AAD, rows and walk hold `entry_aad` took a `tag` that was a byte slice (the data key's ZeroKMS tag), an `s` of type `Tag` that is a selector, and a `j`; it now takes `key_tag`, a `JsonSelector` and `slot`, and its rustdoc says where each comes from. The entry AAD domain is `v1` again: nothing of this format has shipped, so there is no earlier derivation to tell apart. `retrieval` and `retrieval_of` are one `key_retrieval` whose rustdoc says what it builds from the header and the context. On the read side a `Row` is a `ReturnedRow` with a `slot` and a `depth`, `Under` is the `ExpectedSelector` a node must carry, `Placing` is the `ChainWalk` the client runs over a read's rows, and `JsonNode::ordinal` is `slot`, which is what it returns. Sealing asserts the pre-order it relies on. Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- packages/stack-encrypt/src/json/derive.rs | 243 ++++++++++-------- packages/stack-encrypt/src/json/mod.rs | 26 +- packages/stack-encrypt/tests/json_index.rs | 2 +- .../stack-encrypt/tests/json_read_checks.rs | 30 +-- 4 files changed, 157 insertions(+), 144 deletions(-) diff --git a/packages/stack-encrypt/src/json/derive.rs b/packages/stack-encrypt/src/json/derive.rs index f8aed4525..20fdfe71c 100644 --- a/packages/stack-encrypt/src/json/derive.rs +++ b/packages/stack-encrypt/src/json/derive.rs @@ -7,7 +7,7 @@ //! order for an object and in position order for an array. Sealing runs the //! other way, from the leaves up, so that each node's AAD can bind the AEAD //! tags of its children ([`cascade`](super::cascade)); the AAD also binds the -//! slot `j` the entry is stored at. +//! slot the entry is stored at. use std::collections::{BTreeMap, HashMap}; use std::sync::Arc; @@ -43,10 +43,8 @@ const VALUE_DOMAIN: &[u8] = b"stack-encrypt/json/value/v1"; /// PAE domain of a chain head's token: the root's, or a top-level field's /// under the sealed root. const TOKEN_DOMAIN: &[u8] = b"stack-encrypt/json/token/v1"; -/// PAE domain of an entry's AAD. `v2` binds the digest of the node's -/// children's AEAD tags ([`cascade`](super::cascade)); `v1` bound the slot -/// as the node's depth-first position and nothing of its children. -const ENTRY_AAD_DOMAIN: &[u8] = b"stack-encrypt/json/entry/v2"; +/// PAE domain of an entry's AAD ([`entry_aad`]). +const ENTRY_AAD_DOMAIN: &[u8] = b"stack-encrypt/json/entry/v1"; /// PAE domain of an object member's rank under its parent's token, which /// orders the members' ordinals. Keyed, so a token does not let its holder /// test key names. @@ -847,25 +845,29 @@ pub(crate) fn seal_pending<'a, K: 'static>( ) } -/// An entry's AAD: the domain, the keyset, the column's AAD, the key tag, -/// the entry's selector, the slot `j` it is stored at, and the digest of its -/// children's AEAD tags ([`cascade::children_digest`]). The last is what -/// makes a node's ciphertext commit to everything beneath it. +/// The AAD an entry is sealed and opened under, PAE-framed: the domain; +/// `keyset_id`, the keyset the document's data key was minted under; +/// `column`, the column's own AAD (its caller context); `key_tag`, the +/// ZeroKMS tag of that data key ([`JsonHeader::tag`]); `selector`, the +/// entry's path selector; `slot`, the slot the entry is stored at in `sv`; +/// and `children`, the digest of the entry's children's AEAD tags +/// ([`cascade::children_digest`]), which is what makes a node's ciphertext +/// commit to everything beneath it. fn entry_aad( keyset_id: Uuid, - aad: &Context<'_>, - tag: &[u8], - s: &Tag, - j: u64, + column: &Context<'_>, + key_tag: &[u8], + selector: JsonSelector, + slot: u64, children: &ChildrenDigest, ) -> Context<'static> { Context::pae(&[ ENTRY_AAD_DOMAIN, keyset_id.as_bytes(), - aad.as_bytes(), - tag, - s, - &j.to_le_bytes(), + column.as_bytes(), + key_tag, + &selector.0, + &slot.to_le_bytes(), children, ]) } @@ -993,6 +995,10 @@ fn seal( // node has a larger index, and the reverse order seals a node's children // before the node. for (i, plain) in plain.into_iter().enumerate().rev() { + debug_assert!( + children[i].iter().all(|&child| child > i), + "the walk is pre-order: a child is sealed before its parent" + ); // The root's plaintext is the whole document when the root is // sealed; the node encoding it replaces is wiped as it drops. let plaintext = if i == 0 { @@ -1001,12 +1007,15 @@ fn seal( plain }; let digest = cascade::children_digest(children[i].iter().map(|&c| sealed[c].as_slice())); - let c = cipher - .seal( - plaintext, - &entry_aad(keyset_id, aad, &tag, &nodes[i].s, slots[i] as u64, &digest), - ) - .map_err(|_| Error::Aead)?; + let aad = entry_aad( + keyset_id, + aad, + &tag, + JsonSelector(nodes[i].s), + slots[i] as u64, + &digest, + ); + let c = cipher.seal(plaintext, &aad).map_err(|_| Error::Aead)?; sealed[i] = c.as_ref().to_vec(); } let mut entries: Vec> = @@ -1141,22 +1150,27 @@ fn decode_node(plaintext: &Protected>) -> Result { }) } -/// Open one entry stored at slot `j`, under the digest of the children it +/// Open one entry stored at `slot`, under the digest of the children it /// must have been sealed over. The plaintext the cipher returns stays in its /// `Protected` and is wiped when decoding is done. fn open_entry( cipher: &Aes256Cipher, header: &JsonHeader, - aad: &Context<'_>, + column: &Context<'_>, entry: &JsonEntry, - j: u64, + slot: u64, children: &ChildrenDigest, ) -> Result { + let aad = entry_aad( + header.keyset_id, + column, + &header.tag, + JsonSelector(entry.s), + slot, + children, + ); let plaintext = cipher - .open( - LocalCipherText::from(entry.c.clone()), - &entry_aad(header.keyset_id, aad, &header.tag, &entry.s, j, children), - ) + .open(LocalCipherText::from(entry.c.clone()), &aad) .map_err(|_| Error::Aead)?; decode_node(&plaintext) } @@ -1172,25 +1186,22 @@ fn hand_over(value: Protected) -> Value { value.risky_unwrap().into_value() } -/// The retrieve request for a document's one data key, and the AAD its -/// entries open under. -fn retrieval( +/// What opening a document's entries under the caller's context needs +/// from ZeroKMS and from the context: the request that retrieves the +/// document's one data key (its `iv`, its tag and its keyset from the +/// header, and the context's descriptor, which ZeroKMS bound the key to), +/// and the column AAD every entry opens under ([`entry_aad`]), which is the +/// context's AAD. Refused with [`Error::DescriptorTooLong`] before any +/// request is made. +fn key_retrieval( header: &JsonHeader, - context: AeadContext, -) -> Result<(Request, Context<'static>), Error> { - retrieval_of(header, context.into_context()) -} - -/// [`retrieval`] over a context already split into its piece. -fn retrieval_of( - header: &JsonHeader, - piece: vitaminc_aead::ContextPiece<'static>, + context: vitaminc_aead::ContextPiece<'static>, ) -> Result<(Request, Context<'static>), Error> { - let descriptor = Descriptor::from_piece(&piece); + let descriptor = Descriptor::from_piece(&context); descriptor.check()?; Ok(( Request::retrieve_under(header.iv, header.tag.clone(), descriptor, header.keyset_id), - piece.into_aad().into_owned(), + context.into_aad().into_owned(), )) } @@ -1214,7 +1225,7 @@ fn open_document<'a, K>( ) -> Pending<'a, Value, K> { let piece = context.into_context(); let prf_context = piece.clone().into_prf_context().into_owned(); - let (request, aad) = match retrieval_of(&doc.header, piece) { + let (request, aad) = match key_retrieval(&doc.header, piece) { Ok(parts) => parts, Err(error) => return Pending::failed(cipher, error), }; @@ -1288,10 +1299,10 @@ fn open_whole_chain( let rows = entries .into_iter() .enumerate() - .map(|(j, entry)| Row { + .map(|(slot, entry)| ReturnedRow { entry, - j: j as u64, - d: None, + slot: slot as u64, + depth: None, }) .collect(); let reads = [Read { @@ -1356,7 +1367,7 @@ impl DecryptInto for JsonExtracted { return Decryption::failed(wrong("the entry at the path asked for")); } Decryption::from_open(move |cipher| { - let (request, aad) = match retrieval(&node.header, context) { + let (request, aad) = match key_retrieval(&node.header, context.into_context()) { Ok(parts) => parts, Err(error) => return Pending::failed(cipher, error), }; @@ -1368,7 +1379,7 @@ impl DecryptInto for JsonExtracted { &node.header, &aad, &node.entry, - node.j, + node.slot, &cascade::no_children(), )?; opened @@ -1466,13 +1477,13 @@ impl DecryptInto for JsonSubtree { } /// One row of a read as the server returned it, not yet opened: the entry, -/// the slot `j` its ciphertext is bound to, and the depth the server -/// reported, which is optional on the wire and checked against the place -/// the chain puts the row in. -struct Row { +/// the slot its ciphertext is bound to, and the depth the server reported, +/// which is optional on the wire and checked against the place the chain +/// puts the row in. +struct ReturnedRow { entry: JsonEntry, - j: u64, - d: Option, + slot: u64, + depth: Option, } /// Open every row and place each node a token named, checking every row @@ -1497,11 +1508,11 @@ fn open_subtree<'a, K>( // repeating an entry, which would let it line up one container as many // times as it likes: an honest database never returns one, so it is the // wrong entry, not a shape the target refuses. - nodes.sort_by_key(|n| n.j); - if nodes.windows(2).any(|pair| pair[0].j == pair[1].j) { + nodes.sort_by_key(|n| n.slot); + if nodes.windows(2).any(|pair| pair[0].slot == pair[1].slot) { return Pending::failed(cipher, wrong("rows of distinct entries")); } - let (request, aad) = match retrieval(&header, context) { + let (request, aad) = match key_retrieval(&header, context.into_context()) { Ok(parts) => parts, Err(error) => return Pending::failed(cipher, error), }; @@ -1510,10 +1521,10 @@ fn open_subtree<'a, K>( let aes = leaf_cipher(&key_bytes)?; let rows = nodes .into_iter() - .map(|n| Row { + .map(|n| ReturnedRow { entry: n.entry, - j: n.j, - d: n.depth, + slot: n.slot, + depth: n.depth, }) .collect(); let opening = Opening { @@ -1533,12 +1544,12 @@ struct Opening<'o> { aes: &'o Aes256Cipher, } -/// Where a node sits, for the check of its selector: at the top of a read, -/// named by a concrete path, or beneath a node whose selector bytes are -/// known. -enum Under<'p> { - Path(&'p JsonPath), - Node(&'p Protected>), +/// Where the selector a node must carry comes from: the concrete path of +/// the read that named it, for the node at the top of a read, or the +/// selector bytes of its parent, extended by its own label, beneath one. +enum ExpectedSelector<'p> { + OfPath(&'p JsonPath), + UnderParent(&'p Protected>), } /// Put the rows of a read back into the nodes its `reads` named, in the @@ -1576,20 +1587,20 @@ enum Under<'p> { /// under another element (`$.items[0].price` for `$.items[1].price`) at the /// top of a read, which no bound field tells apart; and rows withheld whole. fn place( - rows: Vec, + rows: Vec, reads: &[Read], selectors: &Selectors, bound: u32, opening: &Opening<'_>, ) -> Result>, Error> { - let mut placing = Placing::new(rows, selectors, opening); + let mut walk = ChainWalk::new(rows, selectors, opening); let mut reached = Vec::new(); let mut truncated = false; for read in reads { - let Some(at) = placing.find(&read.token) else { + let Some(at) = walk.find(&read.token) else { continue; }; - let (value, _) = placing.node(at, &read.token, Under::Path(&read.path), 0)?; + let (value, _) = walk.node(at, &read.token, ExpectedSelector::OfPath(&read.path), 0)?; if read.probe { truncated = true; } else { @@ -1598,7 +1609,7 @@ fn place( } // A read returns the subtrees of the nodes its tokens name and nothing // else, so a row none of them reaches is not an answer to it. - if placing.used.contains(&false) { + if walk.placed.contains(&false) { return Err(wrong("rows the walked paths reach")); } if truncated { @@ -1607,36 +1618,37 @@ fn place( Ok(reached) } -/// The rows of one read being placed: which have been placed, and the -/// server's two lookups over them, by self tag and by parent tag. -struct Placing<'k> { - rows: Vec, - used: Vec, - by_x: HashMap, - by_p: HashMap, +/// The client's walk of the chain over the rows of one read, as the server +/// walked it: the rows, which of them the walk has placed, and the server's +/// two lookups over them, by self tag and by parent tag. +struct ChainWalk<'k> { + rows: Vec, + placed: Vec, + by_self_tag: HashMap, + by_parent_tag: HashMap, selectors: &'k Selectors, opening: &'k Opening<'k>, } -impl<'k> Placing<'k> { - fn new(rows: Vec, selectors: &'k Selectors, opening: &'k Opening<'k>) -> Self { +impl<'k> ChainWalk<'k> { + fn new(rows: Vec, selectors: &'k Selectors, opening: &'k Opening<'k>) -> Self { // Where several rows carry one tag, which no sealing produces, the // last of them, as the server's index keeps it // (`eql_v4_internal.json_tree_index` builds it with // `jsonb_object_agg`), so the client tries the row the server found. - let mut by_x = HashMap::with_capacity(rows.len()); - let mut by_p = HashMap::with_capacity(rows.len()); + let mut by_self_tag = HashMap::with_capacity(rows.len()); + let mut by_parent_tag = HashMap::with_capacity(rows.len()); for (i, row) in rows.iter().enumerate() { - let _ = by_x.insert(row.entry.x, i); + let _ = by_self_tag.insert(row.entry.x, i); if let Some(p) = row.entry.p { - let _ = by_p.insert(p, i); + let _ = by_parent_tag.insert(p, i); } } Self { - used: vec![false; rows.len()], + placed: vec![false; rows.len()], rows, - by_x, - by_p, + by_self_tag, + by_parent_tag, selectors, opening, } @@ -1644,7 +1656,7 @@ impl<'k> Placing<'k> { /// The row carrying the self tag of `token`. fn find(&self, token: &Tag) -> Option { - self.by_x + self.by_self_tag .get(&self_tag(&self.opening.header.iv, token)) .copied() } @@ -1660,7 +1672,7 @@ impl<'k> Placing<'k> { // A node has fewer children than there are rows, so the probe loop // is bounded whatever the tags say. for o in 0..u32::try_from(self.rows.len()).unwrap_or(u32::MAX) { - let Some(&child) = self.by_p.get(&parent_tag(iv, token, o)) else { + let Some(&child) = self.by_parent_tag.get(&parent_tag(iv, token, o)) else { break; }; let child_token = hit_token(iv, token, &self.rows[child].entry, o) @@ -1671,11 +1683,11 @@ impl<'k> Placing<'k> { } /// Place and open the row at `at`, the node whose token is `token`, - /// sitting `under` its parent `d` levels below the node its read named: - /// its children found by the chain, then the node opened under the - /// digest of their tags, then the checks, then the children in turn. - /// Returns the node's value and its label, which its parent places it - /// by. + /// `depth` levels below the node its read named, which must carry the + /// selector `expected` says: its children found by the chain, then the + /// node opened under the digest of their tags, then the checks, then + /// the children in turn. Returns the node's value and its label, which + /// its parent places it by. /// /// The recursion follows only nodes that opened, so it follows the tree /// that was sealed, which nests no deeper than [`MAX_DEPTH`]. @@ -1683,16 +1695,16 @@ impl<'k> Placing<'k> { &mut self, at: usize, token: &Tag, - under: Under<'_>, - d: u32, + expected: ExpectedSelector<'_>, + depth: u32, ) -> Result<(Protected, Label), Error> { let refused = || wrong("a node the walked path reaches"); // No row is placed twice: below a node the chain follows only // children that open under its digest, which are the ones sealed // there, and a key's paths are disjoint, so no row is both a read's // node and beneath another's. - self.used[at] = true; - if self.rows[at].d.is_some_and(|got| got != d) { + self.placed[at] = true; + if self.rows[at].depth.is_some_and(|got| got != depth) { return Err(wrong("a row at the depth the chain gives it")); } let children = self.children(token)?; @@ -1703,10 +1715,10 @@ impl<'k> Placing<'k> { ); let row = &self.rows[at]; let Opening { header, aad, aes } = *self.opening; - let opened = open_entry(aes, header, aad, &row.entry, row.j, &digest)?; + let opened = open_entry(aes, header, aad, &row.entry, row.slot, &digest)?; // The sealed root is the whole document, read only at `$`. if opened.whole.is_some() - && !matches!(under, Under::Path(path) if path.segments().is_empty()) + && !matches!(expected, ExpectedSelector::OfPath(path) if path.segments().is_empty()) { return Err(wrong("a node of the walked subtree")); } @@ -1716,14 +1728,16 @@ impl<'k> Placing<'k> { let (label, opened) = take_label(opened); // The selector the chain gives this node: the read's path for the // node it named, its parent's path extended by its own label below. - let bytes = match under { - Under::Path(path) => { + let bytes = match expected { + ExpectedSelector::OfPath(path) => { if !names(path, &label) { return Err(refused()); } path.protected_selector_bytes() } - Under::Node(parent) => child_selector_bytes(parent, &label).ok_or_else(refused)?, + ExpectedSelector::UnderParent(parent) => { + child_selector_bytes(parent, &label).ok_or_else(refused)? + } }; if self.selectors.of(bytes.clone())? != row.entry.s { return Err(refused()); @@ -1735,7 +1749,8 @@ impl<'k> Placing<'k> { Kind::Object => { let mut object = protected_object(); for (child, child_token) in children { - let (value, child_label) = self.child(child, token, &child_token, &bytes, d)?; + let (value, child_label) = + self.child(child, token, &child_token, &bytes, depth)?; let Label::Key(key) = child_label else { return Err(shape("an object member")); }; @@ -1746,7 +1761,8 @@ impl<'k> Placing<'k> { Kind::Array => { let mut array = protected_array(children.len()); for (i, (child, child_token)) in children.into_iter().enumerate() { - let (value, child_label) = self.child(child, token, &child_token, &bytes, d)?; + let (value, child_label) = + self.child(child, token, &child_token, &bytes, depth)?; if !matches!(child_label, Label::Index(at) if at == i as u64) { return Err(shape("an array element in order")); } @@ -1768,9 +1784,14 @@ impl<'k> Placing<'k> { parent_token: &Tag, child_token: &Tag, parent_bytes: &Protected>, - d: u32, + depth: u32, ) -> Result<(Protected, Label), Error> { - let (value, label) = self.node(child, child_token, Under::Node(parent_bytes), d + 1)?; + let (value, label) = self.node( + child, + child_token, + ExpectedSelector::UnderParent(parent_bytes), + depth + 1, + )?; let s = self.rows[child].entry.s; if labelled_token(parent_token, &s, &label).as_ref() != Some(child_token) { return Err(wrong("a child under the rule its label names")); @@ -1876,7 +1897,7 @@ fn open_fallback<'a, K>( let Some(root) = nodes.into_iter().next() else { return Pending::failed(cipher, shape("the sealed root")); }; - let (request, aad) = match retrieval(&root.header, context) { + let (request, aad) = match key_retrieval(&root.header, context.into_context()) { Ok(parts) => parts, Err(error) => return Pending::failed(cipher, error), }; @@ -1888,7 +1909,7 @@ fn open_fallback<'a, K>( &root.header, &aad, &root.entry, - root.j, + root.slot, &cascade::no_children(), )?; let whole = opened.whole.ok_or_else(|| shape("a sealed root"))?; diff --git a/packages/stack-encrypt/src/json/mod.rs b/packages/stack-encrypt/src/json/mod.rs index 805c29f78..c83d2f97a 100644 --- a/packages/stack-encrypt/src/json/mod.rs +++ b/packages/stack-encrypt/src/json/mod.rs @@ -567,14 +567,14 @@ impl JsonDocument { } } - /// The node stored at slot `j` as an extraction returns it: the entry + /// The node stored at `slot` as an extraction returns it: the entry /// with the header beside it. A scalar or an empty container opens on /// its own; a container with children opens only with its subtree. - pub fn node(&self, j: usize) -> Option { - self.entries.get(j).map(|entry| JsonNode { + pub fn node(&self, slot: usize) -> Option { + self.entries.get(slot).map(|entry| JsonNode { entry: entry.clone(), header: self.header.clone(), - j: j as u64, + slot: slot as u64, depth: None, }) } @@ -606,9 +606,9 @@ where } /// One entry as an extraction returns it: the entry, the document's header -/// grafted beside it, and the slot `j` the entry is stored at in the -/// document, which its ciphertext is bound to. What `->` and the walk -/// return per row. +/// grafted beside it, and the slot the entry is stored at in the document +/// (`j` on the wire), which its ciphertext is bound to. What `->` and the +/// walk return per row. /// /// A row is the database's word, so it does not open on its own: the /// selector the client derived opens the entry `->` returned @@ -623,7 +623,7 @@ where pub struct JsonNode { entry: JsonEntry, header: JsonHeader, - j: u64, + slot: u64, depth: Option, } @@ -639,8 +639,8 @@ impl JsonNode { /// The slot the entry is stored at in the document, `j` on the wire. /// The stored order is random, so it is not the node's place in the /// document. - pub fn ordinal(&self) -> u64 { - self.j + pub fn slot(&self) -> u64 { + self.slot } /// The entry's depth below the walk's start, when the walk reported one. pub fn depth(&self) -> Option { @@ -747,7 +747,7 @@ impl JsonSubtree { pub fn is_sealed_root(&self) -> bool { matches!( self.nodes.as_slice(), - [node] if node.j == 0 && node.depth == Some(0) && node.header.sealed_root() + [node] if node.slot == 0 && node.depth == Some(0) && node.header.sealed_root() ) } } @@ -908,7 +908,7 @@ impl From for NodeWire { Self { entry: EntryWire::from(&node.entry), h: base64(&node.header.to_bytes()), - j: node.j, + j: node.slot, d: node.depth, i: None, v: None, @@ -924,7 +924,7 @@ impl TryFrom for JsonNode { entry: JsonEntry::try_from(w.entry)?, header: JsonHeader::from_bytes(&unbase64(&w.h)?) .map_err(|_| WireError("not a header"))?, - j: w.j, + slot: w.j, depth: w.d, }) } diff --git a/packages/stack-encrypt/tests/json_index.rs b/packages/stack-encrypt/tests/json_index.rs index 0859cc2c7..0afd74321 100644 --- a/packages/stack-encrypt/tests/json_index.rs +++ b/packages/stack-encrypt/tests/json_index.rs @@ -1078,7 +1078,7 @@ mod given_an_extracted_node { wire["i"] = json!({"t": "users", "c": "profile"}); wire["v"] = json!(4); let back: JsonNode = serde_json::from_value(wire).unwrap(); - assert_eq!(back.ordinal(), x as u64, "the node keeps its slot"); + assert_eq!(back.slot(), x as u64, "the node keeps its slot"); assert_eq!( cipher .decrypt_as::( diff --git a/packages/stack-encrypt/tests/json_read_checks.rs b/packages/stack-encrypt/tests/json_read_checks.rs index de777ddf0..a93cc53de 100644 --- a/packages/stack-encrypt/tests/json_read_checks.rs +++ b/packages/stack-encrypt/tests/json_read_checks.rs @@ -218,7 +218,7 @@ mod given_the_entry_arrow_returned { let extracted = hello.open(row(&doc, number, None)); assert_eq!(extracted.selector(), hello, "the selector asked for"); assert_eq!( - extracted.node().ordinal(), + extracted.node().slot(), number as u64, "the row as returned" ); @@ -340,7 +340,7 @@ mod given_the_rows_of_a_read { let mut reversed = rows.clone(); reversed.reverse(); let mut by_slot = rows; - by_slot.sort_by_key(JsonNode::ordinal); + by_slot.sort_by_key(JsonNode::slot); for rows in [reversed, by_slot] { let value = open_value(&cipher, nested.open(rows)).await; assert_eq!(value.unwrap(), fixture()["nested"]); @@ -464,10 +464,7 @@ mod given_the_rows_of_a_read { let nested = key(&cipher, "$.nested").await; let deep = slot(&cipher, &doc, "$.nested.deep").await; let rows = walk(&doc, &nested); - let at = rows - .iter() - .position(|n| n.ordinal() == deep as u64) - .unwrap(); + let at = rows.iter().position(|n| n.slot() == deep as u64).unwrap(); let forged = with_x(rows, at, "AAAAAAAAAAA=".into()); assertions::wrong_entry(open_value(&cipher, nested.open(forged)).await, RULE); } @@ -484,7 +481,7 @@ mod given_the_rows_of_a_read { let s = selector_bytes(&selector(&cipher, "$.a.k").await); let as_element = chain::element_token(&a.token_bytes()[0], &s, 0); let rows = walk(&doc, &a); - let at = rows.iter().position(|n| n.ordinal() == k as u64).unwrap(); + let at = rows.iter().position(|n| n.slot() == k as u64).unwrap(); let forged = with_x( rows, at, @@ -602,7 +599,7 @@ mod given_the_rows_of_a_read { let other = slot(&cipher, &doc, "$.nested.list[1].x").await; let honest = walk(&doc, &first); assert_eq!( - honest.iter().map(JsonNode::ordinal).collect::>(), + honest.iter().map(JsonNode::slot).collect::>(), [element as u64, own as u64], "the element and its x" ); @@ -641,10 +638,8 @@ mod given_the_rows_of_a_read { let x = slot(&cipher, &doc, "$.nested.list[0].x").await; let mut rows = walk(&doc, &nested); let (i, k) = ( - rows.iter() - .position(|n| n.ordinal() == deep as u64) - .unwrap(), - rows.iter().position(|n| n.ordinal() == x as u64).unwrap(), + rows.iter().position(|n| n.slot() == deep as u64).unwrap(), + rows.iter().position(|n| n.slot() == x as u64).unwrap(), ); let mut a = serde_json::to_value(&rows[i]).unwrap(); let mut b = serde_json::to_value(&rows[k]).unwrap(); @@ -670,7 +665,7 @@ mod given_the_rows_of_a_read { let rows: Vec = walk(&doc, &nested) .into_iter() .map(|n| { - if n.ordinal() == j as u64 { + if n.slot() == j as u64 { row(&doc, j, Some(d)) } else { n @@ -790,7 +785,7 @@ mod given_the_rows_of_a_read { let nested = key(&cipher, "$.nested").await; let rows: Vec = walk(&doc, &nested) .iter() - .map(|n| row(&doc, n.ordinal() as usize, None)) + .map(|n| row(&doc, n.slot() as usize, None)) .collect(); let value = open_value(&cipher, nested.open(rows)).await; assert_eq!(value.unwrap(), fixture()["nested"], "the sub-document"); @@ -947,10 +942,7 @@ fn with_x(mut rows: Vec, at: usize, x: String) -> Vec { /// `rows` without the row of the entry at slot `j`. fn without(rows: Vec, j: usize) -> Vec { let before = rows.len(); - let rows: Vec = rows - .into_iter() - .filter(|n| n.ordinal() != j as u64) - .collect(); + let rows: Vec = rows.into_iter().filter(|n| n.slot() != j as u64).collect(); assert_eq!(rows.len() + 1, before, "the row at {j} was among them"); rows } @@ -971,7 +963,7 @@ fn exchange_tags(rows: Vec, a: usize, b: usize) -> Vec { let wire = serde_json::to_value(n).unwrap(); (wire["x"].clone(), wire["p"].clone()) }; - let find = |j: usize| rows.iter().position(|n| n.ordinal() == j as u64).unwrap(); + let find = |j: usize| rows.iter().position(|n| n.slot() == j as u64).unwrap(); let (i, k) = (find(a), find(b)); let ((xa, pa), (xb, pb)) = (tags(&rows[i]), tags(&rows[k])); let retag = |n: &JsonNode, x: &Value, p: &Value| { From d959826d5ffc9c9482f2c920bed8b768d5d94542 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 18:25:48 +1100 Subject: [PATCH 16/22] refactor(stack-encrypt): load a root-open document's keyset through the cipher's keyset path A keyset request was answered through a function pointer the cipher captured at `init`, with its own boxed-future aliases per target, so that settling a pending could keep asking only for a `DataKeySource`. Every `StackCipher` is built over a source that is also an `IndexKeySource`, so the settling path now says so and `keyset_state` calls `load_keyset`, the function `StackCipher::keyset` loads a keyset with; the pointer, the aliases and the second constructor go. The `LoadKeyset` request kind stays, and its definition says why a request rather than a call: the open that needs the keyset runs inside a pending's fulfilment, after dispatch, which is the one place a pending reaches ZeroKMS. The load's future is boxed on its own, so the pending's boxed future does not grow by it: a generator is copied into its box with the bytes it does not initialise, and the heap-hygiene spy reads every freed block whole. Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- languages/golang/encrypt/guest/src/ops.rs | 10 +- packages/stack-encrypt/src/cipher.rs | 98 +++++++------------- packages/stack-encrypt/src/dynamic/query.rs | 4 +- packages/stack-encrypt/src/dynamic/term.rs | 4 +- packages/stack-encrypt/src/target/pending.rs | 12 ++- packages/stack-encrypt/src/target/request.rs | 10 +- 6 files changed, 59 insertions(+), 79 deletions(-) diff --git a/languages/golang/encrypt/guest/src/ops.rs b/languages/golang/encrypt/guest/src/ops.rs index aaba4a393..50d273fc1 100644 --- a/languages/golang/encrypt/guest/src/ops.rs +++ b/languages/golang/encrypt/guest/src/ops.rs @@ -51,7 +51,7 @@ use stack_encrypt::sem::MatchOptions; use stack_encrypt::target::IndexSpec; use stack_encrypt::ErrorPayload; use stack_encrypt::{CipherText, KeysetCipher, Label, Node, SealedValue, StackCipherText}; -use stack_kms::DataKeySource; +use stack_kms::{DataKeySource, IndexKeySource}; use zeroize::Zeroizing; use stack_guest_abi::last_error; @@ -101,7 +101,7 @@ pub async fn term( kind: u32, ) -> Result, u32> where - K: DataKeySource + Sync + 'static, + K: DataKeySource + IndexKeySource + Sync + 'static, { // The same proof every stack-encrypt leaf demands: an empty context is // `STATUS_ENCODING` here, before any derivation. @@ -178,7 +178,7 @@ pub async fn encrypt_record( plan: &[u8], ) -> Result, u32> where - K: DataKeySource + Sync + 'static, + K: DataKeySource + IndexKeySource + Sync + 'static, { let plan = parse_plan(plan)?; let tree = @@ -213,7 +213,7 @@ pub async fn decrypt_record( expected: Option