From ca19d714bba8544c621e0698322c989929b05fcd Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 12:10:57 +1100 Subject: [PATCH 01/12] feat(go): store encrypted JSON documents of Go types A field tagged encrypt_into=EncryptedJson may now be a struct, map or slice (or a pointer to one, with null=sql), not only a json.RawMessage. The generated code hands the value to gensupport, which marshals it with encoding/json once, explicitly, and reads the opened text back into the declared type with encrypt.DecodeJSON. Declared numeric fields keep their exact digits, so 64-bit integers and finite float64s round-trip; UseNumber keeps a number inside an interface from becoming a float64. stashgen checks the type when it runs: an interface, an unexported or json:"-" field (both silently dropped), a channel, a scalar or a []byte (base64) is refused with the reason. The query operands become Go values: Contains, ContainedBy, EqAt and the orderings take any. A json.RawMessage crosses unchanged, so existing call sites derive the same query bytes. Methods cannot be generic, and a second method family would be a second way in (principle 6), so any is the one typed form. A plain []byte is refused rather than written as base64. encoding/json writes a zero struct field without omitempty, which narrows a containment query. Contains on a field whose type holds a struct refuses a needle of that type, T or *T: the generator decides which fields hold a struct, and the runtime check is a type switch. Arbitrary needle structs are documented, not walked: refusing them would need reflection over the caller's value and a copy of encoding/json's field rules, and a zero can be meant. Errors from both directions name a Go type and a JSON kind, never the value: encoding/json's own text quotes it. Refs #1351, #1347. Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- languages/golang/cmd/stashgen/main_test.go | 89 +++- languages/golang/encrypt/decode_json_test.go | 57 +++ .../golang/encrypt/gensupport/declaration.go | 6 +- languages/golang/encrypt/gensupport/field.go | 16 + languages/golang/encrypt/gensupport/json.go | 137 +++++- .../golang/encrypt/gensupport/json_test.go | 103 +++++ languages/golang/encrypt/gensupport/widen.go | 12 +- .../internal/testusers/customer_stash.go | 79 ++-- .../internal/testusers/subscriber_stash.go | 389 ++++++++++++++++++ .../encrypt/internal/testusers/subscribers.go | 48 +++ languages/golang/encrypt/json.go | 54 +++ .../encrypt/json_typed_postgres_test.go | 261 ++++++++++++ languages/golang/encrypt/json_typed_test.go | 177 ++++++++ languages/golang/encrypt/term.go | 5 +- languages/golang/stashgen/declaration.go | 8 +- languages/golang/stashgen/emit.go | 51 ++- languages/golang/stashgen/engine.go | 16 +- languages/golang/stashgen/read.go | 138 ++++++- .../testdata/stubsdk/encrypt/encrypt.go | 4 + .../stubsdk/encrypt/gensupport/gensupport.go | 20 +- mise.toml | 6 +- 21 files changed, 1586 insertions(+), 90 deletions(-) create mode 100644 languages/golang/encrypt/decode_json_test.go create mode 100644 languages/golang/encrypt/gensupport/json_test.go create mode 100644 languages/golang/encrypt/internal/testusers/subscriber_stash.go create mode 100644 languages/golang/encrypt/internal/testusers/subscribers.go create mode 100644 languages/golang/encrypt/json_typed_postgres_test.go create mode 100644 languages/golang/encrypt/json_typed_test.go diff --git a/languages/golang/cmd/stashgen/main_test.go b/languages/golang/cmd/stashgen/main_test.go index f2f27d73c..ac6ef0993 100644 --- a/languages/golang/cmd/stashgen/main_test.go +++ b/languages/golang/cmd/stashgen/main_test.go @@ -222,7 +222,7 @@ func TestRunAsksTheEmbeddedEngine(t *testing.T) { if code := run([]string{"-type", "User"}, dir, &stdout, &stderr, stashgen.GuestEngine); code != 1 { t.Fatalf("exit %d, want 1\n%s", code, stderr.String()) } - if !strings.Contains(stderr.String(), "User.Profile") || !strings.Contains(stderr.String(), "EncryptedJson seals a json document: declare the field json.RawMessage") { + if !strings.Contains(stderr.String(), "User.Profile") || !strings.Contains(stderr.String(), "EncryptedJson seals a json document, and []byte is written by encoding/json as a base64 string, not as JSON text; declare the field json.RawMessage") { t.Fatalf("stderr = %q", stderr.String()) } // A context of two segments leaves an EQL field no column: refused by @@ -295,6 +295,87 @@ func TestModelFlagsRepeat(t *testing.T) { // embedded engine with the EQL types: the generated entry has the // document's methods and an accessor per ordered path, and every refusal // names the field before anything is written. +// A document field of a Go type: encoding/json writes it when it is sealed +// and reads it back when it is opened, and the generator refuses, when the +// code is generated, a type that it cannot carry both ways. +func TestRunGeneratesTypedJSONDocuments(t *testing.T) { + checker, err := encrypt.NewChecker(context.Background()) + if err != nil { + t.Skip(err) + } + _ = checker.Close() + types := "type Profile struct {\n\tName string `json:\"name\"`\n\tAge int64 `json:\"age\"`\n\tVisits []Visit `json:\"visits\"`\n}\n\ntype Visit struct {\n\tAt int64 `json:\"at\"`\n}\n\ntype Prefs struct {\n\tTheme string `json:\"theme\"`\n}\n\ntype Hidden struct {\n\tName string\n\tsecret string\n}\n\ntype Loose struct {\n\tExtra any\n}\n\ntype Dropped struct {\n\tName string\n\tToken string `json:\"-\"`\n}\n" + generate := func(fields string) (string, string, int) { + source := "package users\n\nimport (\n\t\"encoding/json\"\n\t\"time\"\n)\n\nvar (\n\t_ json.RawMessage\n\t_ time.Time\n)\n\n" + types + "\ntype Customer struct {\n\t_ struct{} `stash:\"context=customers\"`\n\tID int64 `stash:\"id,passthrough\"`\n" + fields + "\n}\n" + dir := writeModule(t, map[string]string{"model.go": source}) + var stdout, stderr bytes.Buffer + code := run([]string{"-type", "Customer"}, dir, &stdout, &stderr, stashgen.GuestEngine) + written, _ := os.ReadFile(filepath.Join(dir, "customer_stash.go")) + return string(written), stderr.String(), code + } + written, stderr, code := generate("\tProfile Profile `stash:\"profile,encrypt_into=EncryptedJson(ordered=$.age)\"`\n\tPrefs *Prefs `stash:\"prefs,encrypt_into=EncryptedJson,null=sql\"`\n\tLabels map[string]string `stash:\"labels,encrypt_into=EncryptedJson\"`\n\tScores []int64 `stash:\"scores,encrypt_into=EncryptedJson\"`") + if code != 0 { + t.Fatalf("exit %d\n%s", code, stderr) + } + for _, want := range []string{ + "Profile eql.EncryptedJSON", + "Prefs *eql.EncryptedJSON", + `EncryptInto("profile", gensupport.JSON, "EncryptedJson").`, + `EncryptInto("labels", gensupport.JSON, "EncryptedJson").`, + // Written as the field, read back into its type. + `"profile": v.Profile,`, + `"prefs": gensupport.Deref(v.Prefs),`, + `gensupport.GetDocument[Profile](vals, "profile")`, + `gensupport.GetDocumentPtr[Prefs](vals, "prefs")`, + `gensupport.GetDocument[map[string]string](vals, "labels")`, + `gensupport.GetDocument[[]int64](vals, "scores")`, + // A type that holds a struct refuses itself as a needle; a map or + // slice of scalars is a needle like any other. + `Profile: newProfileField(gensupport.NewDocumentField[Profile](declaration, "profile")),`, + `Prefs: newPrefsField(gensupport.NewDocumentField[Prefs](declaration, "prefs")),`, + `Labels: newLabelsField(gensupport.NewField[map[string]string](declaration, "labels")),`, + `Scores: newScoresField(gensupport.NewField[[]int64](declaration, "scores")),`, + "func (f ProfileField) Encrypt(ctx context.Context, c *encrypt.Cipher, v Profile) (eql.EncryptedJSON, error)", + "func (f ProfileField) Contains(ctx context.Context, c *encrypt.Cipher, needle any) (eql.EncryptedJSONQuery, error)", + "A\n// needle of the field's own type, Profile, is refused for that reason.", + "func (p ProfileFieldPath) Gt(ctx context.Context, c *encrypt.Cipher, v any) (eql.EncryptedJSONLeafQuery, error)", + } { + if !strings.Contains(written, want) { + t.Errorf("the generated file lacks %q", want) + } + } + if strings.Contains(written, "\"encoding/json\"") { + t.Error("the generated file imports encoding/json, which no document of a Go type names") + } + if strings.Contains(written, "Labels, is refused") { + t.Error("a map of strings is refused as its own needle") + } + for _, c := range []struct{ name, fields, want string }{ + {"a []byte", "\tProfile []byte `stash:\"profile,encrypt_into=EncryptedJson\"`", "base64 string"}, + {"a string", "\tProfile string `stash:\"profile,encrypt_into=EncryptedJson\"`", "string is one JSON scalar"}, + {"a time", "\tProfile time.Time `stash:\"profile,encrypt_into=EncryptedJson\"`", "is one value, not a document"}, + {"an interface", "\tProfile map[string]any `stash:\"profile,encrypt_into=EncryptedJson\"`", "holds an interface"}, + {"an interface in a struct", "\tProfile Loose `stash:\"profile,encrypt_into=EncryptedJson\"`", "holds an interface"}, + {"an unexported field", "\tProfile Hidden `stash:\"profile,encrypt_into=EncryptedJson\"`", "unexported field secret"}, + {"a field json drops", "\tProfile []Dropped `stash:\"profile,encrypt_into=EncryptedJson\"`", "has a field Token tagged json:\"-\""}, + {"a channel", "\tProfile []chan int `stash:\"profile,encrypt_into=EncryptedJson\"`", "is not a type JSON carries"}, + {"a pointer to a pointer", "\tProfile **Profile `stash:\"profile,encrypt_into=EncryptedJson,null=sql\"`", "a pointer to a pointer"}, + {"a nil document that does not say null=sql", "\tPrefs *Prefs `stash:\"prefs,encrypt_into=EncryptedJson\"`", "null=sql"}, + {"a sealed null", "\tPrefs *Prefs `stash:\"prefs,encrypt_into=EncryptedJson,null=sealed\"`", "Customer.Prefs"}, + {"a struct sealed another way", "\tProfile Profile `stash:\"profile,encrypt\"`", "a sealed field is one scalar"}, + } { + t.Run(c.name, func(t *testing.T) { + written, stderr, code := generate(c.fields) + if code != 1 || written != "" { + t.Fatalf("exit %d, wrote %d bytes\n%s", code, len(written), stderr) + } + if !strings.Contains(stderr, c.want) { + t.Fatalf("stderr = %q, want %q", stderr, c.want) + } + }) + } +} + func TestRunGeneratesEncryptedJSONFields(t *testing.T) { checker, err := encrypt.NewChecker(context.Background()) if err != nil { @@ -320,11 +401,13 @@ func TestRunGeneratesEncryptedJSONFields(t *testing.T) { "Prefs *eql.EncryptedJSON", `EncryptInto("profile", gensupport.JSON, "EncryptedJson").`, `Options("profile", gensupport.TargetOptions{SealedRoot: gensupport.Ptr(false), Ordered: []string{"$.age", "$.visits[*].at"}})`, - "func (f ProfileField) Contains(ctx context.Context, c *encrypt.Cipher, needle json.RawMessage) (eql.EncryptedJSONQuery, error)", + "func (f ProfileField) Contains(ctx context.Context, c *encrypt.Cipher, needle any) (eql.EncryptedJSONQuery, error)", + "func (f ProfileField) EqAt(ctx context.Context, c *encrypt.Cipher, path string, v any) (eql.EncryptedJSONQuery, error)", + `Profile: newProfileField(gensupport.NewField[json.RawMessage](declaration, "profile")),`, "func (f ProfileField) Subtree(ctx context.Context, c *encrypt.Cipher, path string, opts encrypt.JSONReadOptions) (encrypt.JSONRead[eql.EncryptedJSONEntry], error)", "Age ProfileFieldPath // $.age", "VisitsAt ProfileFieldPath // $.visits[*].at", - "func (p ProfileFieldPath) Between(ctx context.Context, c *encrypt.Cipher, lo, hi json.RawMessage) (encrypt.JSONRange[eql.EncryptedJSONLeafQuery], error)", + "func (p ProfileFieldPath) Between(ctx context.Context, c *encrypt.Cipher, lo, hi any) (encrypt.JSONRange[eql.EncryptedJSONLeafQuery], error)", } { if !strings.Contains(written, want) { t.Errorf("the generated file lacks %q", want) diff --git a/languages/golang/encrypt/decode_json_test.go b/languages/golang/encrypt/decode_json_test.go new file mode 100644 index 000000000..4dca067ae --- /dev/null +++ b/languages/golang/encrypt/decode_json_test.go @@ -0,0 +1,57 @@ +package encrypt_test + +import ( + "encoding/json" + "errors" + "math" + "strings" + "testing" + + "github.com/cipherstash/stack/languages/golang/encrypt" +) + +func TestDecodeJSONReadsTheCallersType(t *testing.T) { + type visit struct { + At uint64 `json:"at"` + Place string `json:"place"` + } + v, err := encrypt.DecodeJSON[visit](json.RawMessage(`{"at":18446744073709551615,"place":"home","extra":true}`)) + if err != nil || v != (visit{At: math.MaxUint64, Place: "home"}) { + t.Fatalf("a struct: %+v %v", v, err) + } + // A scalar, as a selector read opens one. + if n, err := encrypt.DecodeJSON[int64](json.RawMessage(`-9223372036854775808`)); err != nil || n != math.MinInt64 { + t.Fatalf("an int64: %d %v", n, err) + } + if f, err := encrypt.DecodeJSON[float64](json.RawMessage(`0.30000000000000004`)); err != nil || f != math.Nextafter(0.3, 1) { + t.Fatalf("a float64: %v %v", f, err) + } + // An interface reads a number as a json.Number, never a float64 that + // would round it. + m, err := encrypt.DecodeJSON[map[string]any](json.RawMessage(`{"big":9007199254740993}`)) + if err != nil || m["big"] != json.Number("9007199254740993") { + t.Fatalf("an interface: %#v %v", m, err) + } + // A JSON null that is there reads as the zero value. + if p, err := encrypt.DecodeJSON[*visit](json.RawMessage(`null`)); err != nil || p != nil { + t.Fatalf("null: %v %v", p, err) + } + for name, text := range map[string]string{ + "an unfound value": ``, + "not JSON": `{"secret-value"`, + "two values": `1 2`, + "a mismatched number": `"secret-value"`, + "a number too big": `123456789012345678901234567890`, + "a fraction for an int": `1.5`, + } { + _, err := encrypt.DecodeJSON[int64](json.RawMessage(text)) + if !errors.Is(err, encrypt.ErrEncoding) { + t.Errorf("%s: %v, want ErrEncoding", name, err) + continue + } + // The error names the Go type and a JSON kind, never the value. + if msg := err.Error(); strings.Contains(msg, "secret") || strings.Contains(msg, "1234567890") || strings.Contains(msg, "1.5") { + t.Errorf("%s: the error shows the value: %s", name, msg) + } + } +} diff --git a/languages/golang/encrypt/gensupport/declaration.go b/languages/golang/encrypt/gensupport/declaration.go index f74c054cf..b3181eae5 100644 --- a/languages/golang/encrypt/gensupport/declaration.go +++ b/languages/golang/encrypt/gensupport/declaration.go @@ -48,8 +48,10 @@ const ( Timestamp Kind = "timestamp" String Kind = "string" Bytes Kind = "bytes" - // JSON is a json.RawMessage field stored as an EncryptedJSON document: - // it crosses as its JSON text, which the engine parses. + // JSON is a field stored as an EncryptedJSON document: it crosses as + // its JSON text, which the engine parses. A json.RawMessage is that + // text; a field of another Go type is marshalled with encoding/json + // ([Field.Encrypt]) and opened with [GetDocument]. JSON Kind = "json" ) diff --git a/languages/golang/encrypt/gensupport/field.go b/languages/golang/encrypt/gensupport/field.go index 7e6d72746..5e2d8deef 100644 --- a/languages/golang/encrypt/gensupport/field.go +++ b/languages/golang/encrypt/gensupport/field.go @@ -15,6 +15,9 @@ type Field[T any] struct { name string plan *record.Plan err error + // holdsStruct is set by [NewDocumentField]: the field is a document + // whose Go type holds a struct, which is no containment needle. + holdsStruct bool } // NewField makes the entry for one sealed field of a declaration. @@ -36,6 +39,19 @@ func NewField[T any](d Declaration, name string) Field[T] { return f } +// NewDocumentField makes the entry for a document field (EncryptedJson) +// whose Go type T holds a struct, as [NewField] does, and its Contains +// refuses a needle of type T or *T: encoding/json writes each struct field +// that has no omitempty or omitzero, zero or not, and a zero one narrows the +// query to the documents holding that zero. stashgen picks it for such a +// field; a json.RawMessage, a map of scalars or a slice of them gets +// NewField. +func NewDocumentField[T any](d Declaration, name string) Field[T] { + f := NewField[T](d, name) + f.holdsStruct = true + return f +} + // kind is the field's declared kind, as its plan holds it. func (f Field[T]) kind() Kind { if pf := f.plan.Field(f.name); pf != nil { diff --git a/languages/golang/encrypt/gensupport/json.go b/languages/golang/encrypt/gensupport/json.go index 717e1821b..9dd3f0342 100644 --- a/languages/golang/encrypt/gensupport/json.go +++ b/languages/golang/encrypt/gensupport/json.go @@ -3,6 +3,7 @@ package gensupport import ( "context" "encoding/json" + "errors" "fmt" "github.com/cipherstash/stack/languages/golang/encrypt" @@ -12,9 +13,85 @@ import ( // The operators of a field stored as an EncryptedJSON document, for the // generated methods of its Fields entry. Each sends the operator's named -// inputs, JSON values as their JSON text ([]byte of the json.RawMessage), -// and checks the engine answered in the encoding and with the number of -// parts the method returns. +// inputs, JSON values as their JSON text ([JSONText]), and checks the +// engine answered in the encoding and with the number of parts the method +// returns. + +// JSONText is a Go value as the JSON text the engine reads: a +// json.RawMessage is JSON text already and crosses as it is (the engine +// parses it, and refuses one that is not JSON); any other value is +// marshalled with encoding/json, explicitly and once, so a declared integer +// field is written as its exact digits and a float64 as the shortest text +// that reads back to it. A plain []byte is refused: encoding/json would +// write it as a base64 string, which is never what JSON text in a []byte +// means; convert it, json.RawMessage(b). +// +// An error wraps encrypt.ErrEncoding and names a Go type, never the value: +// encoding/json's own text quotes a NaN, and a MarshalJSON's error may hold +// anything. +func JSONText(v any) ([]byte, error) { + switch v := v.(type) { + case json.RawMessage: + return v, nil + case []byte: + return nil, fmt.Errorf("%w: a []byte is not JSON text to encoding/json, which writes it as a base64 string; convert JSON text to json.RawMessage", encrypt.ErrEncoding) + } + text, err := json.Marshal(v) + if err == nil { + return text, nil + } + var typeErr *json.UnsupportedTypeError + var valueErr *json.UnsupportedValueError + var marshalerErr *json.MarshalerError + switch { + case errors.As(err, &typeErr): + return nil, fmt.Errorf("%w: encoding/json cannot write a %s, in a %T", encrypt.ErrEncoding, typeErr.Type, v) + case errors.As(err, &valueErr): + return nil, fmt.Errorf("%w: a %T holds a value encoding/json cannot write: a NaN, an infinity, or a cycle", encrypt.ErrEncoding, v) + case errors.As(err, &marshalerErr): + return nil, fmt.Errorf("%w: the MarshalJSON or MarshalText of %s failed, in a %T", encrypt.ErrEncoding, marshalerErr.Type, v) + } + return nil, fmt.Errorf("%w: a %T does not write as JSON", encrypt.ErrEncoding, v) +} + +// GetDocument reads one opened document field into the Go type T the struct +// declares, with [encrypt.DecodeJSON]: the explicit encoding/json read that +// mirrors the write [JSONText] made. A json.RawMessage field is read with +// [Get] instead, as the text it is. +func GetDocument[T any](vals Values, name string) (T, error) { + var out T + v, ok := vals[name] + if !ok { + return out, fmt.Errorf("gensupport: the opened value has no field %q", name) + } + text, ok := v.([]byte) + if !ok { + return out, fmt.Errorf("gensupport: field %q: %w: the document opened as %T, not its JSON text", name, encrypt.ErrEncoding, v) + } + out, err := encrypt.DecodeJSON[T](text) + if err != nil { + return out, fmt.Errorf("gensupport: field %q: %w", name, err) + } + return out, nil +} + +// GetDocumentPtr reads one opened document field declared over a pointer +// (null=sql): nil for a field that opened as nil, the SQL NULL it was +// stored as, else a pointer to the value as [GetDocument] reads it. +func GetDocumentPtr[T any](vals Values, name string) (*T, error) { + v, ok := vals[name] + if !ok { + return nil, fmt.Errorf("gensupport: the opened value has no field %q", name) + } + if v == nil { + return nil, nil + } + out, err := GetDocument[T](vals, name) + if err != nil { + return nil, err + } + return &out, nil +} // jsonTerm derives one JSON operator's term on the field: an exact term in // encoding, of parts byte strings ("eql") or texts ("text"); parts 0 takes @@ -40,10 +117,33 @@ func (f Field[T]) jsonTerm(ctx context.Context, c *encrypt.Cipher, operator, enc return term, nil } +// jsonOperand is one JSON operand as its text ([JSONText]), with the +// field's name on an error. +func (f Field[T]) jsonOperand(v any) ([]byte, error) { + text, err := JSONText(v) + if err != nil { + return nil, fmt.Errorf("gensupport: %s: %w", f.name, err) + } + return text, nil +} + // JSONQuery derives a containment operator's query on a JSON field -// (contains, contained_by): the needle's projection, Q. -func JSONQuery[Q ~[]byte, T any](ctx context.Context, f Field[T], c *encrypt.Cipher, operator string, needle json.RawMessage) (Q, error) { - term, err := f.jsonTerm(ctx, c, operator, "eql", 1, vcvalue.Object{{Key: "json", Value: []byte(needle)}}) +// (contains, contained_by): the needle's projection, Q. The needle is any +// value [JSONText] writes. Contains on a field of [NewDocumentField] +// refuses a needle of the field's own type, T or *T: its zero-valued +// fields would narrow the query. +func JSONQuery[Q ~[]byte, T any](ctx context.Context, f Field[T], c *encrypt.Cipher, operator string, needle any) (Q, error) { + if f.holdsStruct && operator == "contains" { + switch needle.(type) { + case T, *T: + return nil, fmt.Errorf("%w: gensupport: %s: Contains takes a needle, not the field's own type %T: encoding/json writes each of its struct fields that has no omitempty, and a zero one narrows the query to documents holding that zero. Use a map, or a needle type whose fields are pointers or carry omitempty", encrypt.ErrEncoding, f.name, needle) + } + } + text, err := f.jsonOperand(needle) + if err != nil { + return nil, err + } + term, err := f.jsonTerm(ctx, c, operator, "eql", 1, vcvalue.Object{{Key: "json", Value: text}}) if err != nil { return nil, err } @@ -52,9 +152,14 @@ func JSONQuery[Q ~[]byte, T any](ctx context.Context, f Field[T], c *encrypt.Cip // JSONQueryAt derives an operator at a path on a JSON field (eq_at, lt_at, // lte_at, gt_at, gte_at): Q is the needle for eq_at and the leaf operand -// for the orderings. -func JSONQueryAt[Q ~[]byte, T any](ctx context.Context, f Field[T], c *encrypt.Cipher, operator, path string, v json.RawMessage) (Q, error) { - term, err := f.jsonTerm(ctx, c, operator, "eql", 1, vcvalue.Object{{Key: "path", Value: path}, {Key: "json", Value: []byte(v)}}) +// for the orderings. v is any value [JSONText] writes: a string, a number, +// a bool, nil for JSON null, or a json.RawMessage. +func JSONQueryAt[Q ~[]byte, T any](ctx context.Context, f Field[T], c *encrypt.Cipher, operator, path string, v any) (Q, error) { + text, err := f.jsonOperand(v) + if err != nil { + return nil, err + } + term, err := f.jsonTerm(ctx, c, operator, "eql", 1, vcvalue.Object{{Key: "path", Value: path}, {Key: "json", Value: text}}) if err != nil { return nil, err } @@ -62,9 +167,17 @@ func JSONQueryAt[Q ~[]byte, T any](ctx context.Context, f Field[T], c *encrypt.C } // JSONBetweenAt derives between_at on a JSON field: the two leaf operands, -// low then high, inclusive. -func JSONBetweenAt[Q ~[]byte, T any](ctx context.Context, f Field[T], c *encrypt.Cipher, path string, lo, hi json.RawMessage) (encrypt.JSONRange[Q], error) { - term, err := f.jsonTerm(ctx, c, "between_at", "eql", 2, vcvalue.Object{{Key: "path", Value: path}, {Key: "low", Value: []byte(lo)}, {Key: "high", Value: []byte(hi)}}) +// low then high, inclusive, each any value [JSONText] writes. +func JSONBetweenAt[Q ~[]byte, T any](ctx context.Context, f Field[T], c *encrypt.Cipher, path string, lo, hi any) (encrypt.JSONRange[Q], error) { + low, err := f.jsonOperand(lo) + if err != nil { + return encrypt.JSONRange[Q]{}, err + } + high, err := f.jsonOperand(hi) + if err != nil { + return encrypt.JSONRange[Q]{}, err + } + term, err := f.jsonTerm(ctx, c, "between_at", "eql", 2, vcvalue.Object{{Key: "path", Value: path}, {Key: "low", Value: low}, {Key: "high", Value: high}}) if err != nil { return encrypt.JSONRange[Q]{}, err } diff --git a/languages/golang/encrypt/gensupport/json_test.go b/languages/golang/encrypt/gensupport/json_test.go new file mode 100644 index 000000000..7fe4ca768 --- /dev/null +++ b/languages/golang/encrypt/gensupport/json_test.go @@ -0,0 +1,103 @@ +package gensupport + +import ( + "encoding/json" + "errors" + "math" + "strings" + "testing" + + "github.com/cipherstash/stack/languages/golang/encrypt" +) + +// numbers holds the values a float64 would round: the 64-bit integer +// extremes and a float that needs 17 significant digits. +type numbers struct { + Min int64 `json:"min"` + Max int64 `json:"max"` + UMax uint64 `json:"umax"` + Float float64 `json:"float"` +} + +var extremes = numbers{Min: math.MinInt64, Max: math.MaxInt64, UMax: math.MaxUint64, Float: 0.30000000000000004} // the float64 after 0.3: 17 digits + +// failingMarshaler's MarshalJSON fails with an error that holds a value. +type failingMarshaler struct{} + +func (failingMarshaler) MarshalJSON() ([]byte, error) { return nil, errors.New("secret-value") } + +func TestJSONTextWritesExactNumbersAndRefusesWhatIsNotJSON(t *testing.T) { + text, err := JSONText(extremes) + if err != nil { + t.Fatal(err) + } + // encoding/json writes integers as their digits and a float64 as the + // shortest text that reads back to it: 0.30000000000000004, 17 digits. + want := `{"min":-9223372036854775808,"max":9223372036854775807,"umax":18446744073709551615,"float":0.30000000000000004}` + if string(text) != want { + t.Fatalf("JSONText = %s, want %s", text, want) + } + // A json.RawMessage is JSON text already: it crosses unchanged, spacing + // and all, so a RawMessage needle sends what it sent before. + raw := json.RawMessage(`{ "a" : 1 }`) + if text, err := JSONText(raw); err != nil || string(text) != string(raw) { + t.Fatalf("a RawMessage: %s %v", text, err) + } + // Scalars for the path operators, and nil for JSON null. + for v, want := range map[any]string{"Alice": `"Alice"`, int64(40): `40`, true: `true`, nil: `null`} { + if text, err := JSONText(v); err != nil || string(text) != want { + t.Errorf("JSONText(%v) = %s %v, want %s", v, text, err, want) + } + } + for name, v := range map[string]any{ + "[]byte": []byte(`{"secret-value":1}`), + "NaN": math.NaN(), + "infinity": map[string]float64{"x": math.Inf(1)}, + "channel": make(chan int), + "MarshalJSON": failingMarshaler{}, + } { + _, err := JSONText(v) + if !errors.Is(err, encrypt.ErrEncoding) { + t.Errorf("%s: %v, want ErrEncoding", name, err) + continue + } + // The error names a type, never the value: encoding/json's own + // text would say "unsupported value: NaN" or "+Inf". + if msg := err.Error(); strings.Contains(msg, "secret-value") || strings.Contains(msg, "unsupported value") || strings.Contains(msg, "+Inf") { + t.Errorf("%s: the error shows the value: %s", name, msg) + } + } +} + +func TestGetDocumentReadsTheDeclaredType(t *testing.T) { + text, err := JSONText(extremes) + if err != nil { + t.Fatal(err) + } + vals := Values{"doc": text, "none": nil, "wrong": "text"} + got, err := GetDocument[numbers](vals, "doc") + if err != nil || got != extremes { + t.Fatalf("GetDocument = %+v %v, want %+v", got, err, extremes) + } + ptr, err := GetDocumentPtr[numbers](vals, "doc") + if err != nil || ptr == nil || *ptr != extremes { + t.Fatalf("GetDocumentPtr = %v %v", ptr, err) + } + // A null=sql field that was nil opened as nil, and reads as nil. + if ptr, err := GetDocumentPtr[numbers](vals, "none"); err != nil || ptr != nil { + t.Fatalf("a nil document: %v %v", ptr, err) + } + if _, err := GetDocument[numbers](vals, "missing"); err == nil { + t.Fatal("a field the opened value lacks reads") + } + if _, err := GetDocument[numbers](vals, "wrong"); !errors.Is(err, encrypt.ErrEncoding) { + t.Fatalf("a document that opened as a string: %v", err) + } + // A document that does not fit the type is refused, naming the type + // and the field, not the value. + vals["doc"] = []byte(`{"max":"secret-value"}`) + _, err = GetDocument[numbers](vals, "doc") + if !errors.Is(err, encrypt.ErrEncoding) || strings.Contains(err.Error(), "secret-value") || !strings.Contains(err.Error(), `"doc"`) { + t.Fatalf("a mismatched document: %v", err) + } +} diff --git a/languages/golang/encrypt/gensupport/widen.go b/languages/golang/encrypt/gensupport/widen.go index ca9f0564e..7d96af747 100644 --- a/languages/golang/encrypt/gensupport/widen.go +++ b/languages/golang/encrypt/gensupport/widen.go @@ -1,7 +1,6 @@ package gensupport import ( - "encoding/json" "fmt" "math" "reflect" @@ -34,14 +33,9 @@ func asKind(kind Kind, v any) (any, error) { } if kind == JSON { // A document crosses as its JSON text, in a bytes value; the - // engine parses it. - switch raw := v.(type) { - case json.RawMessage: - return []byte(raw), nil - case []byte: - return raw, nil - } - return nil, fmt.Errorf("%w: a JSON field takes a json.RawMessage, not a %T", encrypt.ErrEncoding, v) + // engine parses it. A json.RawMessage is that text; a field of + // another Go type (a struct, say) is marshalled, explicitly, here. + return JSONText(v) } if !isInteger(kind) { return v, nil diff --git a/languages/golang/encrypt/internal/testusers/customer_stash.go b/languages/golang/encrypt/internal/testusers/customer_stash.go index 50b266aa1..2b1ebe9fb 100644 --- a/languages/golang/encrypt/internal/testusers/customer_stash.go +++ b/languages/golang/encrypt/internal/testusers/customer_stash.go @@ -139,36 +139,44 @@ func (f CustomerProfileField) Encrypt(ctx context.Context, c *encrypt.Cipher, v return eql.EncryptedJSON(out.EQL), err } -// Contains derives the query for documents that contain needle, a JSON value: -// col @> query. Each array element of the needle must sit inside one element of -// the document's array. -func (f CustomerProfileField) Contains(ctx context.Context, c *encrypt.Cipher, needle json.RawMessage) (eql.EncryptedJSONQuery, error) { +// Contains derives the query for documents that contain needle: col @> query. +// needle is a json.RawMessage of JSON text, or any value encoding/json writes, +// such as a map or a struct. Each member it writes must be in the document, so +// a zero-valued struct field narrows the query to documents holding that zero: +// give a needle struct pointer fields, or omitempty, or use a map. Each array +// element of the needle must sit inside one element of the document's array. +func (f CustomerProfileField) Contains(ctx context.Context, c *encrypt.Cipher, needle any) (eql.EncryptedJSONQuery, error) { return gensupport.JSONQuery[eql.EncryptedJSONQuery](ctx, f.field, c, "contains", needle) } -// ContainedBy derives the query for documents contained by needle, a JSON -// value: col <@ query. -func (f CustomerProfileField) ContainedBy(ctx context.Context, c *encrypt.Cipher, needle json.RawMessage) (eql.EncryptedJSONQuery, error) { +// ContainedBy derives the query for documents contained by needle: col <@ +// query. needle is a json.RawMessage of JSON text, or any value encoding/json +// writes; here a zero-valued field widens the query, admitting documents that +// hold that zero. +func (f CustomerProfileField) ContainedBy(ctx context.Context, c *encrypt.Cipher, needle any) (eql.EncryptedJSONQuery, error) { return gensupport.JSONQuery[eql.EncryptedJSONQuery](ctx, f.field, c, "contained_by", needle) } // EqAt derives the query for documents holding the scalar v at path: col @> -// query. A wildcard step matches when any element holds it; a position ([2]) is -// refused, since the stored selector reads every position as the wildcard. -func (f CustomerProfileField) EqAt(ctx context.Context, c *encrypt.Cipher, path string, v json.RawMessage) (eql.EncryptedJSONQuery, error) { +// query. v is a Go string, number or bool, nil for JSON null, or a +// json.RawMessage of JSON text. A wildcard step matches when any element holds +// it; a position ([2]) is refused, since the stored selector reads every +// position as the wildcard. +func (f CustomerProfileField) EqAt(ctx context.Context, c *encrypt.Cipher, path string, v any) (eql.EncryptedJSONQuery, error) { return gensupport.JSONQueryAt[eql.EncryptedJSONQuery](ctx, f.field, c, "eq_at", path, v) } // Selector prepares `->` at path: bind Selector() as col -> $1::text, and open -// the entries it returns with Open. A position ([2]) is refused; read one with -// Subtree. +// the entries it returns with Open; encrypt.DecodeJSON reads each value into a +// Go type. A position ([2]) is refused; read one with Subtree. func (f CustomerProfileField) Selector(ctx context.Context, c *encrypt.Cipher, path string) (encrypt.JSONSelection[eql.EncryptedJSONEntry], error) { return gensupport.JSONSelect[eql.EncryptedJSONEntry](ctx, f.field, c, path) } // Subtree prepares a read of the value at path: bind Tokens() as // eql_v4.json_tree_subtree(col, $1::text[], budget), and open the rows it -// returns with Open or OpenOne. +// returns with Open or OpenOne; encrypt.DecodeJSON reads each value into a Go +// type. func (f CustomerProfileField) Subtree(ctx context.Context, c *encrypt.Cipher, path string, opts encrypt.JSONReadOptions) (encrypt.JSONRead[eql.EncryptedJSONEntry], error) { return gensupport.JSONSubtree[eql.EncryptedJSONEntry](ctx, f.field, c, path, opts) } @@ -194,6 +202,7 @@ type CustomerProfileFieldAt struct { // CustomerProfileFieldPath is one ordered path of Profile: its orderings // compare the first value `->` extracts there, by the leaf's ordering term. +// Each operand is a Go number or string, or a json.RawMessage of JSON text. // Numbers order as float64, so integers past 2^53 can tie; text orders case- // and accent-folded. type CustomerProfileFieldPath struct { @@ -201,23 +210,23 @@ type CustomerProfileFieldPath struct { path string } -func (p CustomerProfileFieldPath) Lt(ctx context.Context, c *encrypt.Cipher, v json.RawMessage) (eql.EncryptedJSONLeafQuery, error) { +func (p CustomerProfileFieldPath) Lt(ctx context.Context, c *encrypt.Cipher, v any) (eql.EncryptedJSONLeafQuery, error) { return gensupport.JSONQueryAt[eql.EncryptedJSONLeafQuery](ctx, p.field, c, "lt_at", p.path, v) } -func (p CustomerProfileFieldPath) Lte(ctx context.Context, c *encrypt.Cipher, v json.RawMessage) (eql.EncryptedJSONLeafQuery, error) { +func (p CustomerProfileFieldPath) Lte(ctx context.Context, c *encrypt.Cipher, v any) (eql.EncryptedJSONLeafQuery, error) { return gensupport.JSONQueryAt[eql.EncryptedJSONLeafQuery](ctx, p.field, c, "lte_at", p.path, v) } -func (p CustomerProfileFieldPath) Gt(ctx context.Context, c *encrypt.Cipher, v json.RawMessage) (eql.EncryptedJSONLeafQuery, error) { +func (p CustomerProfileFieldPath) Gt(ctx context.Context, c *encrypt.Cipher, v any) (eql.EncryptedJSONLeafQuery, error) { return gensupport.JSONQueryAt[eql.EncryptedJSONLeafQuery](ctx, p.field, c, "gt_at", p.path, v) } -func (p CustomerProfileFieldPath) Gte(ctx context.Context, c *encrypt.Cipher, v json.RawMessage) (eql.EncryptedJSONLeafQuery, error) { +func (p CustomerProfileFieldPath) Gte(ctx context.Context, c *encrypt.Cipher, v any) (eql.EncryptedJSONLeafQuery, error) { return gensupport.JSONQueryAt[eql.EncryptedJSONLeafQuery](ctx, p.field, c, "gte_at", p.path, v) } -func (p CustomerProfileFieldPath) Between(ctx context.Context, c *encrypt.Cipher, lo, hi json.RawMessage) (encrypt.JSONRange[eql.EncryptedJSONLeafQuery], error) { +func (p CustomerProfileFieldPath) Between(ctx context.Context, c *encrypt.Cipher, lo, hi any) (encrypt.JSONRange[eql.EncryptedJSONLeafQuery], error) { return gensupport.JSONBetweenAt[eql.EncryptedJSONLeafQuery](ctx, p.field, c, p.path, lo, hi) } @@ -241,36 +250,44 @@ func (f CustomerPrefsField) Encrypt(ctx context.Context, c *encrypt.Cipher, v js return eql.EncryptedJSON(out.EQL), err } -// Contains derives the query for documents that contain needle, a JSON value: -// col @> query. Each array element of the needle must sit inside one element of -// the document's array. -func (f CustomerPrefsField) Contains(ctx context.Context, c *encrypt.Cipher, needle json.RawMessage) (eql.EncryptedJSONQuery, error) { +// Contains derives the query for documents that contain needle: col @> query. +// needle is a json.RawMessage of JSON text, or any value encoding/json writes, +// such as a map or a struct. Each member it writes must be in the document, so +// a zero-valued struct field narrows the query to documents holding that zero: +// give a needle struct pointer fields, or omitempty, or use a map. Each array +// element of the needle must sit inside one element of the document's array. +func (f CustomerPrefsField) Contains(ctx context.Context, c *encrypt.Cipher, needle any) (eql.EncryptedJSONQuery, error) { return gensupport.JSONQuery[eql.EncryptedJSONQuery](ctx, f.field, c, "contains", needle) } -// ContainedBy derives the query for documents contained by needle, a JSON -// value: col <@ query. -func (f CustomerPrefsField) ContainedBy(ctx context.Context, c *encrypt.Cipher, needle json.RawMessage) (eql.EncryptedJSONQuery, error) { +// ContainedBy derives the query for documents contained by needle: col <@ +// query. needle is a json.RawMessage of JSON text, or any value encoding/json +// writes; here a zero-valued field widens the query, admitting documents that +// hold that zero. +func (f CustomerPrefsField) ContainedBy(ctx context.Context, c *encrypt.Cipher, needle any) (eql.EncryptedJSONQuery, error) { return gensupport.JSONQuery[eql.EncryptedJSONQuery](ctx, f.field, c, "contained_by", needle) } // EqAt derives the query for documents holding the scalar v at path: col @> -// query. A wildcard step matches when any element holds it; a position ([2]) is -// refused, since the stored selector reads every position as the wildcard. -func (f CustomerPrefsField) EqAt(ctx context.Context, c *encrypt.Cipher, path string, v json.RawMessage) (eql.EncryptedJSONQuery, error) { +// query. v is a Go string, number or bool, nil for JSON null, or a +// json.RawMessage of JSON text. A wildcard step matches when any element holds +// it; a position ([2]) is refused, since the stored selector reads every +// position as the wildcard. +func (f CustomerPrefsField) EqAt(ctx context.Context, c *encrypt.Cipher, path string, v any) (eql.EncryptedJSONQuery, error) { return gensupport.JSONQueryAt[eql.EncryptedJSONQuery](ctx, f.field, c, "eq_at", path, v) } // Selector prepares `->` at path: bind Selector() as col -> $1::text, and open -// the entries it returns with Open. A position ([2]) is refused; read one with -// Subtree. +// the entries it returns with Open; encrypt.DecodeJSON reads each value into a +// Go type. A position ([2]) is refused; read one with Subtree. func (f CustomerPrefsField) Selector(ctx context.Context, c *encrypt.Cipher, path string) (encrypt.JSONSelection[eql.EncryptedJSONEntry], error) { return gensupport.JSONSelect[eql.EncryptedJSONEntry](ctx, f.field, c, path) } // Subtree prepares a read of the value at path: bind Tokens() as // eql_v4.json_tree_subtree(col, $1::text[], budget), and open the rows it -// returns with Open or OpenOne. +// returns with Open or OpenOne; encrypt.DecodeJSON reads each value into a Go +// type. func (f CustomerPrefsField) Subtree(ctx context.Context, c *encrypt.Cipher, path string, opts encrypt.JSONReadOptions) (encrypt.JSONRead[eql.EncryptedJSONEntry], error) { return gensupport.JSONSubtree[eql.EncryptedJSONEntry](ctx, f.field, c, path, opts) } diff --git a/languages/golang/encrypt/internal/testusers/subscriber_stash.go b/languages/golang/encrypt/internal/testusers/subscriber_stash.go new file mode 100644 index 000000000..58b189841 --- /dev/null +++ b/languages/golang/encrypt/internal/testusers/subscriber_stash.go @@ -0,0 +1,389 @@ +// Code generated by stashgen. DO NOT EDIT. + +package testusers + +import ( + "context" + "log/slog" + + "github.com/cipherstash/stack/languages/golang/encrypt" + "github.com/cipherstash/stack/languages/golang/encrypt/eql" + "github.com/cipherstash/stack/languages/golang/encrypt/gensupport" +) + +// Stops compiling when the library does not accept this version of generated file. +const _ = gensupport.GeneratedVersion2 + +// Subscriber prints its sealed fields in the clear: it has no String or +// LogValue method. Write them, or run stashgen with -redact. + +type EncryptedSubscriber struct { + ID int64 + Profile eql.EncryptedJSON + Prefs *eql.EncryptedJSON + Labels eql.EncryptedJSON +} + +func (e EncryptedSubscriber) String() string { + return gensupport.Redacted("EncryptedSubscriber", map[string]any{"ID": e.ID}, "Profile", "Prefs", "Labels") +} + +func (e EncryptedSubscriber) LogValue() slog.Value { + return gensupport.RedactedLog(map[string]any{"ID": e.ID}, "Profile", "Prefs", "Labels") +} + +// Stops compiling when Subscriber gains, loses, reorders or retypes a field. +var _ = subscriberShape(Subscriber{}) + +type subscriberShape struct { + _ struct{} + ID int64 + Profile SubscriberProfile + Prefs *SubscriberPrefs + Labels map[string]string +} + +var subscriberDeclaration = gensupport.Declare("subscribers"). + Passthrough("id"). + EncryptInto("profile", gensupport.JSON, "EncryptedJson"). + Options("profile", gensupport.TargetOptions{Ordered: []string{"$.age", "$.visits[*].at"}}). + EncryptInto("prefs", gensupport.JSON, "EncryptedJson"). + Null("prefs", gensupport.NullSQL). + EncryptInto("labels", gensupport.JSON, "EncryptedJson") + +var subscriberCodec = gensupport.New(gensupport.Generated[Subscriber, EncryptedSubscriber]{ + TypeName: "Subscriber", + Declaration: subscriberDeclaration, + PrintsPlaintext: true, + Source: func(v Subscriber) gensupport.Values { + return gensupport.Values{ + "id": v.ID, + "profile": v.Profile, + "prefs": gensupport.Deref(v.Prefs), + "labels": v.Labels, + } + }, + Seal: func(rec gensupport.Record) (EncryptedSubscriber, error) { + var e EncryptedSubscriber + var err error + if e.ID, err = gensupport.Passthrough[int64](rec, "id"); err != nil { + return EncryptedSubscriber{}, err + } + e.Profile = eql.EncryptedJSON(rec["profile"].EQL) + if o, ok := rec["prefs"]; ok { + v := eql.EncryptedJSON(o.EQL) + e.Prefs = &v + } + e.Labels = eql.EncryptedJSON(rec["labels"].EQL) + return e, nil + }, + Open: func(e EncryptedSubscriber) gensupport.Record { + rec := gensupport.Record{ + "id": {Value: e.ID}, + "profile": {EQL: e.Profile}, + "labels": {EQL: e.Labels}, + } + if e.Prefs != nil { + rec["prefs"] = gensupport.Output{EQL: *e.Prefs} + } + return rec + }, + Value: func(e EncryptedSubscriber, vals gensupport.Values) (Subscriber, error) { + var v Subscriber + var err error + if v.ID, err = gensupport.Get[int64](vals, "id"); err != nil { + return Subscriber{}, err + } + if v.Profile, err = gensupport.GetDocument[SubscriberProfile](vals, "profile"); err != nil { + return Subscriber{}, err + } + if v.Prefs, err = gensupport.GetDocumentPtr[SubscriberPrefs](vals, "prefs"); err != nil { + return Subscriber{}, err + } + if v.Labels, err = gensupport.GetDocument[map[string]string](vals, "labels"); err != nil { + return Subscriber{}, err + } + return v, nil + }, +}) + +// EncryptSubscriber seals each Subscriber, with one ZeroKMS request for each +// 500 sealed values, plus one the first time a keyset is used. The result has +// one element for each input, in the same order. +func EncryptSubscriber(ctx context.Context, cipher *encrypt.Cipher, values []Subscriber) ([]EncryptedSubscriber, error) { + return subscriberCodec.Encrypt(ctx, cipher, values) +} + +// DecryptSubscriber opens each EncryptedSubscriber, with one ZeroKMS request +// for each 500 sealed values, plus one the first time a keyset is used. +func DecryptSubscriber(ctx context.Context, d encrypt.Decrypter, encrypted []EncryptedSubscriber) ([]Subscriber, error) { + return subscriberCodec.Decrypt(ctx, d, encrypted) +} + +var SubscriberFields = struct { + Profile SubscriberProfileField + Prefs SubscriberPrefsField + Labels SubscriberLabelsField +}{ + Profile: newSubscriberProfileField(gensupport.NewDocumentField[SubscriberProfile](subscriberDeclaration, "profile")), + Prefs: newSubscriberPrefsField(gensupport.NewDocumentField[SubscriberPrefs](subscriberDeclaration, "prefs")), + Labels: newSubscriberLabelsField(gensupport.NewField[map[string]string](subscriberDeclaration, "labels")), +} + +type SubscriberProfileField struct { + field gensupport.Field[SubscriberProfile] + // At has one accessor per path the field orders, whose methods + // are the orderings there. + At SubscriberProfileFieldAt +} + +func newSubscriberProfileField(f gensupport.Field[SubscriberProfile]) SubscriberProfileField { + return SubscriberProfileField{field: f, At: SubscriberProfileFieldAt{ + Age: SubscriberProfileFieldPath{field: f, path: "$.age"}, + VisitsAt: SubscriberProfileFieldPath{field: f, path: "$.visits[*].at"}, + }} +} + +func (f SubscriberProfileField) Encrypt(ctx context.Context, c *encrypt.Cipher, v SubscriberProfile) (eql.EncryptedJSON, error) { + out, err := f.field.Encrypt(ctx, c, v) + return eql.EncryptedJSON(out.EQL), err +} + +// Contains derives the query for documents that contain needle: col @> query. +// needle is a json.RawMessage of JSON text, or any value encoding/json writes, +// such as a map or a struct. Each member it writes must be in the document, so +// a zero-valued struct field narrows the query to documents holding that zero: +// give a needle struct pointer fields, or omitempty, or use a map. Each array +// element of the needle must sit inside one element of the document's array. A +// needle of the field's own type, SubscriberProfile, is refused for that +// reason. +func (f SubscriberProfileField) Contains(ctx context.Context, c *encrypt.Cipher, needle any) (eql.EncryptedJSONQuery, error) { + return gensupport.JSONQuery[eql.EncryptedJSONQuery](ctx, f.field, c, "contains", needle) +} + +// ContainedBy derives the query for documents contained by needle: col <@ +// query. needle is a json.RawMessage of JSON text, or any value encoding/json +// writes; here a zero-valued field widens the query, admitting documents that +// hold that zero. +func (f SubscriberProfileField) ContainedBy(ctx context.Context, c *encrypt.Cipher, needle any) (eql.EncryptedJSONQuery, error) { + return gensupport.JSONQuery[eql.EncryptedJSONQuery](ctx, f.field, c, "contained_by", needle) +} + +// EqAt derives the query for documents holding the scalar v at path: col @> +// query. v is a Go string, number or bool, nil for JSON null, or a +// json.RawMessage of JSON text. A wildcard step matches when any element holds +// it; a position ([2]) is refused, since the stored selector reads every +// position as the wildcard. +func (f SubscriberProfileField) EqAt(ctx context.Context, c *encrypt.Cipher, path string, v any) (eql.EncryptedJSONQuery, error) { + return gensupport.JSONQueryAt[eql.EncryptedJSONQuery](ctx, f.field, c, "eq_at", path, v) +} + +// Selector prepares `->` at path: bind Selector() as col -> $1::text, and open +// the entries it returns with Open; encrypt.DecodeJSON reads each value into a +// Go type. A position ([2]) is refused; read one with Subtree. +func (f SubscriberProfileField) Selector(ctx context.Context, c *encrypt.Cipher, path string) (encrypt.JSONSelection[eql.EncryptedJSONEntry], error) { + return gensupport.JSONSelect[eql.EncryptedJSONEntry](ctx, f.field, c, path) +} + +// Subtree prepares a read of the value at path: bind Tokens() as +// eql_v4.json_tree_subtree(col, $1::text[], budget), and open the rows it +// returns with Open or OpenOne; encrypt.DecodeJSON reads each value into a Go +// type. +func (f SubscriberProfileField) Subtree(ctx context.Context, c *encrypt.Cipher, path string, opts encrypt.JSONReadOptions) (encrypt.JSONRead[eql.EncryptedJSONEntry], error) { + return gensupport.JSONSubtree[eql.EncryptedJSONEntry](ctx, f.field, c, path, opts) +} + +// Children derives the token of the node at path, a path with no wildcard, +// which eql_v4.json_tree_children(col, $1::text) takes. Its rows are encrypted +// entries one level down. +func (f SubscriberProfileField) Children(ctx context.Context, c *encrypt.Cipher, path string) (encrypt.JSONToken, error) { + return gensupport.JSONNodeToken(ctx, f.field, c, "children", path) +} + +// ChildCount derives the token of the node at path, which +// eql_v4.json_tree_child_count(col, $1::text) takes: zero for a scalar or an +// empty container, NULL where the row has no node there or for the sealed root. +func (f SubscriberProfileField) ChildCount(ctx context.Context, c *encrypt.Cipher, path string) (encrypt.JSONToken, error) { + return gensupport.JSONNodeToken(ctx, f.field, c, "child_count", path) +} + +type SubscriberProfileFieldAt struct { + Age SubscriberProfileFieldPath // $.age + VisitsAt SubscriberProfileFieldPath // $.visits[*].at +} + +// SubscriberProfileFieldPath is one ordered path of Profile: its orderings +// compare the first value `->` extracts there, by the leaf's ordering term. +// Each operand is a Go number or string, or a json.RawMessage of JSON text. +// Numbers order as float64, so integers past 2^53 can tie; text orders case- +// and accent-folded. +type SubscriberProfileFieldPath struct { + field gensupport.Field[SubscriberProfile] + path string +} + +func (p SubscriberProfileFieldPath) Lt(ctx context.Context, c *encrypt.Cipher, v any) (eql.EncryptedJSONLeafQuery, error) { + return gensupport.JSONQueryAt[eql.EncryptedJSONLeafQuery](ctx, p.field, c, "lt_at", p.path, v) +} + +func (p SubscriberProfileFieldPath) Lte(ctx context.Context, c *encrypt.Cipher, v any) (eql.EncryptedJSONLeafQuery, error) { + return gensupport.JSONQueryAt[eql.EncryptedJSONLeafQuery](ctx, p.field, c, "lte_at", p.path, v) +} + +func (p SubscriberProfileFieldPath) Gt(ctx context.Context, c *encrypt.Cipher, v any) (eql.EncryptedJSONLeafQuery, error) { + return gensupport.JSONQueryAt[eql.EncryptedJSONLeafQuery](ctx, p.field, c, "gt_at", p.path, v) +} + +func (p SubscriberProfileFieldPath) Gte(ctx context.Context, c *encrypt.Cipher, v any) (eql.EncryptedJSONLeafQuery, error) { + return gensupport.JSONQueryAt[eql.EncryptedJSONLeafQuery](ctx, p.field, c, "gte_at", p.path, v) +} + +func (p SubscriberProfileFieldPath) Between(ctx context.Context, c *encrypt.Cipher, lo, hi any) (encrypt.JSONRange[eql.EncryptedJSONLeafQuery], error) { + return gensupport.JSONBetweenAt[eql.EncryptedJSONLeafQuery](ctx, p.field, c, p.path, lo, hi) +} + +// Selector prepares `->` at this path: the value an ordering here compares is +// col -> Selector(), so bind it beside the ordering's operand, from the same +// path. +func (p SubscriberProfileFieldPath) Selector(ctx context.Context, c *encrypt.Cipher) (encrypt.JSONSelection[eql.EncryptedJSONEntry], error) { + return gensupport.JSONSelect[eql.EncryptedJSONEntry](ctx, p.field, c, p.path) +} + +type SubscriberPrefsField struct { + field gensupport.Field[SubscriberPrefs] +} + +func newSubscriberPrefsField(f gensupport.Field[SubscriberPrefs]) SubscriberPrefsField { + return SubscriberPrefsField{field: f} +} + +func (f SubscriberPrefsField) Encrypt(ctx context.Context, c *encrypt.Cipher, v SubscriberPrefs) (eql.EncryptedJSON, error) { + out, err := f.field.Encrypt(ctx, c, v) + return eql.EncryptedJSON(out.EQL), err +} + +// Contains derives the query for documents that contain needle: col @> query. +// needle is a json.RawMessage of JSON text, or any value encoding/json writes, +// such as a map or a struct. Each member it writes must be in the document, so +// a zero-valued struct field narrows the query to documents holding that zero: +// give a needle struct pointer fields, or omitempty, or use a map. Each array +// element of the needle must sit inside one element of the document's array. A +// needle of the field's own type, SubscriberPrefs, is refused for that reason. +func (f SubscriberPrefsField) Contains(ctx context.Context, c *encrypt.Cipher, needle any) (eql.EncryptedJSONQuery, error) { + return gensupport.JSONQuery[eql.EncryptedJSONQuery](ctx, f.field, c, "contains", needle) +} + +// ContainedBy derives the query for documents contained by needle: col <@ +// query. needle is a json.RawMessage of JSON text, or any value encoding/json +// writes; here a zero-valued field widens the query, admitting documents that +// hold that zero. +func (f SubscriberPrefsField) ContainedBy(ctx context.Context, c *encrypt.Cipher, needle any) (eql.EncryptedJSONQuery, error) { + return gensupport.JSONQuery[eql.EncryptedJSONQuery](ctx, f.field, c, "contained_by", needle) +} + +// EqAt derives the query for documents holding the scalar v at path: col @> +// query. v is a Go string, number or bool, nil for JSON null, or a +// json.RawMessage of JSON text. A wildcard step matches when any element holds +// it; a position ([2]) is refused, since the stored selector reads every +// position as the wildcard. +func (f SubscriberPrefsField) EqAt(ctx context.Context, c *encrypt.Cipher, path string, v any) (eql.EncryptedJSONQuery, error) { + return gensupport.JSONQueryAt[eql.EncryptedJSONQuery](ctx, f.field, c, "eq_at", path, v) +} + +// Selector prepares `->` at path: bind Selector() as col -> $1::text, and open +// the entries it returns with Open; encrypt.DecodeJSON reads each value into a +// Go type. A position ([2]) is refused; read one with Subtree. +func (f SubscriberPrefsField) Selector(ctx context.Context, c *encrypt.Cipher, path string) (encrypt.JSONSelection[eql.EncryptedJSONEntry], error) { + return gensupport.JSONSelect[eql.EncryptedJSONEntry](ctx, f.field, c, path) +} + +// Subtree prepares a read of the value at path: bind Tokens() as +// eql_v4.json_tree_subtree(col, $1::text[], budget), and open the rows it +// returns with Open or OpenOne; encrypt.DecodeJSON reads each value into a Go +// type. +func (f SubscriberPrefsField) Subtree(ctx context.Context, c *encrypt.Cipher, path string, opts encrypt.JSONReadOptions) (encrypt.JSONRead[eql.EncryptedJSONEntry], error) { + return gensupport.JSONSubtree[eql.EncryptedJSONEntry](ctx, f.field, c, path, opts) +} + +// Children derives the token of the node at path, a path with no wildcard, +// which eql_v4.json_tree_children(col, $1::text) takes. Its rows are encrypted +// entries one level down. +func (f SubscriberPrefsField) Children(ctx context.Context, c *encrypt.Cipher, path string) (encrypt.JSONToken, error) { + return gensupport.JSONNodeToken(ctx, f.field, c, "children", path) +} + +// ChildCount derives the token of the node at path, which +// eql_v4.json_tree_child_count(col, $1::text) takes: zero for a scalar or an +// empty container, NULL where the row has no node there or for the sealed root. +func (f SubscriberPrefsField) ChildCount(ctx context.Context, c *encrypt.Cipher, path string) (encrypt.JSONToken, error) { + return gensupport.JSONNodeToken(ctx, f.field, c, "child_count", path) +} + +type SubscriberLabelsField struct { + field gensupport.Field[map[string]string] +} + +func newSubscriberLabelsField(f gensupport.Field[map[string]string]) SubscriberLabelsField { + return SubscriberLabelsField{field: f} +} + +func (f SubscriberLabelsField) Encrypt(ctx context.Context, c *encrypt.Cipher, v map[string]string) (eql.EncryptedJSON, error) { + out, err := f.field.Encrypt(ctx, c, v) + return eql.EncryptedJSON(out.EQL), err +} + +// Contains derives the query for documents that contain needle: col @> query. +// needle is a json.RawMessage of JSON text, or any value encoding/json writes, +// such as a map or a struct. Each member it writes must be in the document, so +// a zero-valued struct field narrows the query to documents holding that zero: +// give a needle struct pointer fields, or omitempty, or use a map. Each array +// element of the needle must sit inside one element of the document's array. +func (f SubscriberLabelsField) Contains(ctx context.Context, c *encrypt.Cipher, needle any) (eql.EncryptedJSONQuery, error) { + return gensupport.JSONQuery[eql.EncryptedJSONQuery](ctx, f.field, c, "contains", needle) +} + +// ContainedBy derives the query for documents contained by needle: col <@ +// query. needle is a json.RawMessage of JSON text, or any value encoding/json +// writes; here a zero-valued field widens the query, admitting documents that +// hold that zero. +func (f SubscriberLabelsField) ContainedBy(ctx context.Context, c *encrypt.Cipher, needle any) (eql.EncryptedJSONQuery, error) { + return gensupport.JSONQuery[eql.EncryptedJSONQuery](ctx, f.field, c, "contained_by", needle) +} + +// EqAt derives the query for documents holding the scalar v at path: col @> +// query. v is a Go string, number or bool, nil for JSON null, or a +// json.RawMessage of JSON text. A wildcard step matches when any element holds +// it; a position ([2]) is refused, since the stored selector reads every +// position as the wildcard. +func (f SubscriberLabelsField) EqAt(ctx context.Context, c *encrypt.Cipher, path string, v any) (eql.EncryptedJSONQuery, error) { + return gensupport.JSONQueryAt[eql.EncryptedJSONQuery](ctx, f.field, c, "eq_at", path, v) +} + +// Selector prepares `->` at path: bind Selector() as col -> $1::text, and open +// the entries it returns with Open; encrypt.DecodeJSON reads each value into a +// Go type. A position ([2]) is refused; read one with Subtree. +func (f SubscriberLabelsField) Selector(ctx context.Context, c *encrypt.Cipher, path string) (encrypt.JSONSelection[eql.EncryptedJSONEntry], error) { + return gensupport.JSONSelect[eql.EncryptedJSONEntry](ctx, f.field, c, path) +} + +// Subtree prepares a read of the value at path: bind Tokens() as +// eql_v4.json_tree_subtree(col, $1::text[], budget), and open the rows it +// returns with Open or OpenOne; encrypt.DecodeJSON reads each value into a Go +// type. +func (f SubscriberLabelsField) Subtree(ctx context.Context, c *encrypt.Cipher, path string, opts encrypt.JSONReadOptions) (encrypt.JSONRead[eql.EncryptedJSONEntry], error) { + return gensupport.JSONSubtree[eql.EncryptedJSONEntry](ctx, f.field, c, path, opts) +} + +// Children derives the token of the node at path, a path with no wildcard, +// which eql_v4.json_tree_children(col, $1::text) takes. Its rows are encrypted +// entries one level down. +func (f SubscriberLabelsField) Children(ctx context.Context, c *encrypt.Cipher, path string) (encrypt.JSONToken, error) { + return gensupport.JSONNodeToken(ctx, f.field, c, "children", path) +} + +// ChildCount derives the token of the node at path, which +// eql_v4.json_tree_child_count(col, $1::text) takes: zero for a scalar or an +// empty container, NULL where the row has no node there or for the sealed root. +func (f SubscriberLabelsField) ChildCount(ctx context.Context, c *encrypt.Cipher, path string) (encrypt.JSONToken, error) { + return gensupport.JSONNodeToken(ctx, f.field, c, "child_count", path) +} diff --git a/languages/golang/encrypt/internal/testusers/subscribers.go b/languages/golang/encrypt/internal/testusers/subscribers.go new file mode 100644 index 000000000..63ccc5c4a --- /dev/null +++ b/languages/golang/encrypt/internal/testusers/subscribers.go @@ -0,0 +1,48 @@ +package testusers + +//go:generate go run github.com/cipherstash/stack/languages/golang/cmd/stashgen -type Subscriber -name Subscriber + +// Subscriber holds encrypted JSON documents (the EQL type EncryptedJson) of Go +// types: encoding/json writes each to JSON text when it is sealed and reads +// it back when it is opened. Its profile orders $.age and each visit's $.at, +// its preferences are SQL NULL when nil, and its labels are a map. +type Subscriber struct { + _ struct{} `stash:"context=subscribers"` + ID int64 `stash:"id,passthrough"` + Profile SubscriberProfile `stash:"profile,encrypt_into=EncryptedJson(ordered=$.age;$.visits[*].at)"` + Prefs *SubscriberPrefs `stash:"prefs,encrypt_into=EncryptedJson,null=sql"` + Labels map[string]string `stash:"labels,encrypt_into=EncryptedJson"` +} + +// SubscriberProfile is a subscriber's profile document. Big, Small and Score hold +// the values a float64 would round: they cross as their exact digits. +type SubscriberProfile struct { + Name string `json:"name"` + Age int64 `json:"age"` + Tier string `json:"tier,omitempty"` + Big uint64 `json:"big"` + Small int64 `json:"small"` + Score float64 `json:"score"` + Visits []Visit `json:"visits"` +} + +// Visit is one visit in a profile. +type Visit struct { + At int64 `json:"at"` + Place string `json:"place"` +} + +// SubscriberPrefs is a subscriber's optional preferences document. +type SubscriberPrefs struct { + Theme string `json:"theme"` +} + +// ProfileNeedle is a containment needle for a subscriber's profile: every +// field is a pointer with omitempty, so a field left nil is not in the +// needle and does not narrow the query. +type ProfileNeedle struct { + Name *string `json:"name,omitempty"` + Age *int64 `json:"age,omitempty"` + Tier *string `json:"tier,omitempty"` + Visits []Visit `json:"visits,omitempty"` +} diff --git a/languages/golang/encrypt/json.go b/languages/golang/encrypt/json.go index 37ed84527..eea5c0f69 100644 --- a/languages/golang/encrypt/json.go +++ b/languages/golang/encrypt/json.go @@ -1,10 +1,13 @@ package encrypt import ( + "bytes" "context" "database/sql/driver" "encoding/json" + "errors" "fmt" + "io" "strings" "github.com/cipherstash/stack/languages/golang/internal/record" @@ -324,3 +327,54 @@ func (r JSONRead[E]) OpenOne(ctx context.Context, rows []E) (JSONValue, error) { return JSONValue{}, fmt.Errorf("%w: the read reached %d values, not one; open it with Open", ErrEncoding, len(values)) } } + +// DecodeJSON reads JSON text a read opened, a [JSONValue]'s JSON or one +// value of [JSONRead.Open], into a value of the caller's type T, with +// encoding/json's rules: a number reads into a declared integer field +// exactly, int64 and uint64 included, and into an interface (any) as a +// json.Number, never a float64 that rounds it. A member T does not declare +// is skipped, as encoding/json skips it. Text that is not one JSON value +// (empty, as an unfound JSONValue's is, or followed by more) is refused. +// +// An error wraps [ErrEncoding] and names T and the kind of JSON value that +// did not fit, never the value: what a read opened is plaintext. +func DecodeJSON[T any](text json.RawMessage) (T, error) { + var out T + dec := json.NewDecoder(bytes.NewReader(text)) + dec.UseNumber() + if err := dec.Decode(&out); err != nil { + var zero T + return zero, decodeError(err, out) + } + // One value, and nothing after it: Decode reads the first of a stream. + if _, err := dec.Token(); !errors.Is(err, io.EOF) { + var zero T + return zero, fmt.Errorf("%w: the JSON text holds more than one value, and reads into one %T", ErrEncoding, zero) + } + return out, nil +} + +// decodeError is encoding/json's decode error without the text it quotes: +// a SyntaxError names the character it met and an UnmarshalTypeError the +// number it could not store, both plaintext. It keeps the kind of JSON value +// and the Go type that did not fit. +func decodeError(err error, out any) error { + var typeErr *json.UnmarshalTypeError + switch { + case errors.As(err, &typeErr): + // Value is "number", "number 1.5", "string", "bool", "array" or + // "object": its first word is the kind, the rest the value. + kind, _, _ := strings.Cut(typeErr.Value, " ") + return fmt.Errorf("%w: a JSON %s does not read into the Go type %s, in a %T", ErrEncoding, kind, typeErr.Type, out) + case errors.Is(err, io.EOF), errors.Is(err, io.ErrUnexpectedEOF): + return fmt.Errorf("%w: no JSON value to read into a %T (an unfound JSONValue holds none: check Found)", ErrEncoding, out) + } + var syntax *json.SyntaxError + if errors.As(err, &syntax) { + return fmt.Errorf("%w: the text is not JSON (at byte %d), and reads into no %T", ErrEncoding, syntax.Offset, out) + } + // An UnmarshalJSON or UnmarshalText of the caller's type failed, or + // encoding/json refused the target (a non-pointer, say): its text may + // hold the value, so only the type is named. + return fmt.Errorf("%w: the JSON text does not read into a %T", ErrEncoding, out) +} diff --git a/languages/golang/encrypt/json_typed_postgres_test.go b/languages/golang/encrypt/json_typed_postgres_test.go new file mode 100644 index 000000000..d51ba57e1 --- /dev/null +++ b/languages/golang/encrypt/json_typed_postgres_test.go @@ -0,0 +1,261 @@ +package encrypt_test + +import ( + "context" + "math" + "os" + "reflect" + "slices" + "testing" + + "github.com/cipherstash/stack/languages/golang/encrypt" + "github.com/cipherstash/stack/languages/golang/encrypt/eql" + "github.com/cipherstash/stack/languages/golang/encrypt/internal/testusers" +) + +// Encrypted JSON documents of Go types (testusers.Subscriber) inside +// PostgreSQL with EQL v4 installed: stored from structs, found by queries +// whose needles and operands are Go values, and opened back into Go types, +// whole and in part. Skipped unless DATABASE_URL is set. +func TestJSONTypedDocumentsInPostgres(t *testing.T) { + url := os.Getenv("DATABASE_URL") + if url == "" { + t.Skip("DATABASE_URL is not set") + } + ctx := context.Background() + c := deterministicEQLClient(t) + cipher := c.DefaultKeyset() + conn := eqlPostgresConn(t, ctx, url) + exec := func(q string, args ...any) { + t.Helper() + if _, err := conn.ExecContext(ctx, q, args...); err != nil { + t.Fatalf("%s: %v", q, err) + } + } + exec(`CREATE TEMP TABLE subscribers (id bigint PRIMARY KEY, profile public.eql_v4_json_tree NOT NULL, prefs public.eql_v4_json_tree, labels public.eql_v4_json_tree NOT NULL)`) + t.Cleanup(func() { _, _ = conn.ExecContext(context.Background(), `DROP TABLE IF EXISTS subscribers`) }) + encrypted, err := testusers.EncryptSubscriber(ctx, cipher, subscribers) + if err != nil { + t.Fatal(err) + } + for _, e := range encrypted { + exec(`INSERT INTO subscribers VALUES ($1, $2, $3, $4)`, e.ID, e.Profile, e.Prefs, e.Labels) + } + ids := func(q string, args ...any) []int64 { + t.Helper() + rows, err := conn.QueryContext(ctx, q, args...) + if err != nil { + t.Fatalf("%s: %v", q, err) + } + defer rows.Close() + var out []int64 + for rows.Next() { + var id int64 + if err := rows.Scan(&id); err != nil { + t.Fatal(err) + } + out = append(out, id) + } + if err := rows.Err(); err != nil { + t.Fatal(err) + } + return out + } + want := func(name string, got []int64, want ...int64) { + t.Helper() + if !slices.Equal(got, want) { + t.Errorf("%s: rows %v, want %v", name, got, want) + } + } + must := func(v []byte, err error) []byte { + t.Helper() + if err != nil { + t.Fatal(err) + } + return v + } + f := testusers.SubscriberFields.Profile + + // Stored rows, re-serialised by PostgreSQL, open into the declared + // types: every 64-bit integer and the 17-digit float exactly. + rows, err := conn.QueryContext(ctx, `SELECT id, profile, prefs, labels FROM subscribers ORDER BY id`) + if err != nil { + t.Fatal(err) + } + var stored []testusers.EncryptedSubscriber + for rows.Next() { + var e testusers.EncryptedSubscriber + if err := rows.Scan(&e.ID, &e.Profile, &e.Prefs, &e.Labels); err != nil { + t.Fatal(err) + } + stored = append(stored, e) + } + if err := rows.Err(); err != nil { + t.Fatal(err) + } + _ = rows.Close() + back, err := testusers.DecryptSubscriber(ctx, c, stored) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(back, subscribers) { + t.Fatalf("read back:\n got %+v\nwant %+v", back, subscribers) + } + + // Containment with a needle struct of pointer and omitempty fields: a + // field left nil is not in the needle. + contains := `SELECT id FROM subscribers WHERE profile @> $1::eql_v4.query_json_tree ORDER BY id` + want("contains tier", ids(contains, eql.EncryptedJSONQuery(must(f.Contains(ctx, cipher, testusers.ProfileNeedle{Tier: ptr("gold")})))), 1) + want("contains name and age", ids(contains, eql.EncryptedJSONQuery(must(f.Contains(ctx, cipher, testusers.ProfileNeedle{Name: ptr("Bob"), Age: ptr(int64(7))})))), 2) + // A map needle names only what it holds; the needle struct's Visit has + // no omitempty, so its zero Place is in the needle and narrows it: no + // visit at 3 has an empty place. The narrowing the docs warn of. + want("contains a visit, map", ids(contains, eql.EncryptedJSONQuery(must(f.Contains(ctx, cipher, map[string]any{"visits": []map[string]any{{"at": 3}}})))), 1) + want("contains a visit, zero place", ids(contains, eql.EncryptedJSONQuery(must(f.Contains(ctx, cipher, testusers.ProfileNeedle{Visits: []testusers.Visit{{At: 3}}}))))) + // Contained by the document type itself: its zero fields widen, so + // Bob's whole profile contains no other row's. + want("contained by", ids(`SELECT id FROM subscribers WHERE profile <@ $1::eql_v4.query_json_tree ORDER BY id`, eql.EncryptedJSONQuery(must(f.ContainedBy(ctx, cipher, subscribers[1].Profile)))), 2) + // Equality at a path with Go scalars, the integer extremes included. + want("eq_at umax", ids(contains, eql.EncryptedJSONQuery(must(f.EqAt(ctx, cipher, "$.big", uint64(math.MaxUint64))))), 1) + want("eq_at min", ids(contains, eql.EncryptedJSONQuery(must(f.EqAt(ctx, cipher, "$.small", int64(math.MinInt64))))), 1) + want("eq_at name", ids(contains, eql.EncryptedJSONQuery(must(f.EqAt(ctx, cipher, "$.name", "Carol")))), 3) + want("eq_at a visit", ids(contains, eql.EncryptedJSONQuery(must(f.EqAt(ctx, cipher, "$.visits[*].at", 5)))), 2) + // The optional and the map columns. + want("prefs contains", ids(`SELECT id FROM subscribers WHERE prefs @> $1::eql_v4.query_json_tree ORDER BY id`, eql.EncryptedJSONQuery(must(testusers.SubscriberFields.Prefs.Contains(ctx, cipher, map[string]string{"theme": "dark"})))), 1) + want("labels contains", ids(`SELECT id FROM subscribers WHERE labels @> $1::eql_v4.query_json_tree ORDER BY id`, eql.EncryptedJSONQuery(must(testusers.SubscriberFields.Labels.Contains(ctx, cipher, map[string]string{"team": "red"})))), 1) + + // Orderings with Go numbers. + age, err := f.At.Age.Selector(ctx, cipher) + if err != nil { + t.Fatal(err) + } + gt, err := f.At.Age.Gt(ctx, cipher, 30) + if err != nil { + t.Fatal(err) + } + want("age > 30", ids(`SELECT id FROM subscribers WHERE (profile -> $1::text) > $2::eql_v4.query_json_tree_leaf ORDER BY id`, age.Selector(), gt), 1, 3) + rng, err := f.At.Age.Between(ctx, cipher, int64(5), 40.5) + if err != nil { + t.Fatal(err) + } + want("age between", ids(`SELECT id FROM subscribers WHERE (profile -> $1::text) >= $2::eql_v4.query_json_tree_leaf AND (profile -> $1::text) <= $3::eql_v4.query_json_tree_leaf ORDER BY id`, age.Selector(), rng.Low, rng.High), 1, 2) + + // `->` opened, and each value decoded into a Go type. + selected := func(sel encrypt.JSONSelection[eql.EncryptedJSONEntry]) []encrypt.JSONValue { + t.Helper() + rows, err := conn.QueryContext(ctx, `SELECT profile -> $1::text FROM subscribers ORDER BY id`, sel.Selector()) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + var entries []*eql.EncryptedJSONEntry + for rows.Next() { + var e *eql.EncryptedJSONEntry + if err := rows.Scan(&e); err != nil { + t.Fatal(err) + } + entries = append(entries, e) + } + if err := rows.Err(); err != nil { + t.Fatal(err) + } + values, err := sel.Open(ctx, entries) + if err != nil { + t.Fatal(err) + } + return values + } + big, err := f.Selector(ctx, cipher, "$.big") + if err != nil { + t.Fatal(err) + } + var bigs []uint64 + for _, v := range selected(big) { + n, err := encrypt.DecodeJSON[uint64](v.JSON) + if err != nil || !v.Found { + t.Fatalf("$.big: %+v %v", v, err) + } + bigs = append(bigs, n) + } + if !slices.Equal(bigs, []uint64{math.MaxUint64, math.MaxInt64, 0}) { + t.Errorf("$.big: %v", bigs) + } + score, err := f.Selector(ctx, cipher, "$.score") + if err != nil { + t.Fatal(err) + } + if s, err := encrypt.DecodeJSON[float64](selected(score)[0].JSON); err != nil || s != 0.30000000000000004 { + t.Errorf("$.score: %v %v", s, err) + } + // The sealed root at $ opens to the whole document, which decodes into + // the field's own type. + root, err := f.Selector(ctx, cipher, "$") + if err != nil { + t.Fatal(err) + } + for i, v := range selected(root) { + p, err := encrypt.DecodeJSON[testusers.SubscriberProfile](v.JSON) + if err != nil || !reflect.DeepEqual(p, subscribers[i].Profile) { + t.Errorf("$ row %d: %+v %v", i, p, err) + } + } + + // A subtree read, each value decoded: every visit of each row, then one + // row's visits as one slice. + subtree := func(read encrypt.JSONRead[eql.EncryptedJSONEntry], id int64) []eql.EncryptedJSONEntry { + t.Helper() + rows, err := conn.QueryContext(ctx, `SELECT e FROM subscribers, eql_v4.json_tree_subtree(profile, $2::text[]) AS e WHERE id = $1`, id, read.Tokens()) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + var entries []eql.EncryptedJSONEntry + for rows.Next() { + var e eql.EncryptedJSONEntry + if err := rows.Scan(&e); err != nil { + t.Fatal(err) + } + entries = append(entries, e) + } + if err := rows.Err(); err != nil { + t.Fatal(err) + } + return entries + } + each, err := f.Subtree(ctx, cipher, "$.visits[*]", encrypt.JSONReadOptions{}) + if err != nil { + t.Fatal(err) + } + groups := make([][]eql.EncryptedJSONEntry, len(subscribers)) + for i, s := range subscribers { + groups[i] = subtree(each, s.ID) + } + opened, err := each.Open(ctx, groups) + if err != nil { + t.Fatal(err) + } + for i, values := range opened { + var visits []testusers.Visit + for _, text := range values { + v, err := encrypt.DecodeJSON[testusers.Visit](text) + if err != nil { + t.Fatal(err) + } + visits = append(visits, v) + } + if len(visits) != len(subscribers[i].Profile.Visits) || (len(visits) > 0 && !reflect.DeepEqual(visits, subscribers[i].Profile.Visits)) { + t.Errorf("visits of row %d: %+v", i, visits) + } + } + all, err := f.Subtree(ctx, cipher, "$.visits", encrypt.JSONReadOptions{}) + if err != nil { + t.Fatal(err) + } + one, err := all.OpenOne(ctx, subtree(all, 1)) + if err != nil || !one.Found { + t.Fatalf("OpenOne $.visits: %+v %v", one, err) + } + if visits, err := encrypt.DecodeJSON[[]testusers.Visit](one.JSON); err != nil || !reflect.DeepEqual(visits, subscribers[0].Profile.Visits) { + t.Errorf("$.visits as a slice: %+v %v", visits, err) + } +} diff --git a/languages/golang/encrypt/json_typed_test.go b/languages/golang/encrypt/json_typed_test.go new file mode 100644 index 000000000..8249053d2 --- /dev/null +++ b/languages/golang/encrypt/json_typed_test.go @@ -0,0 +1,177 @@ +package encrypt_test + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "math" + "reflect" + "strings" + "testing" + + "github.com/cipherstash/stack/languages/golang/encrypt" + "github.com/cipherstash/stack/languages/golang/encrypt/internal/testusers" +) + +// Generated code for document fields of Go types: testusers.Subscriber's +// Profile (a struct, ordered at $.age and $.visits[*].at), its optional +// Prefs (a pointer to a struct, null=sql) and its Labels (a map). + +// subscribers hold the values a float64 would round, in declared fields. +var subscribers = []testusers.Subscriber{ + { + ID: 1, + Profile: testusers.SubscriberProfile{ + Name: "Alice", Age: 34, Tier: "gold", + Big: math.MaxUint64, Small: math.MinInt64, Score: 0.30000000000000004, + Visits: []testusers.Visit{{At: 3, Place: "home"}, {At: 1, Place: "work"}}, + }, + Prefs: &testusers.SubscriberPrefs{Theme: "dark"}, + Labels: map[string]string{"team": "red"}, + }, + {ID: 2, Profile: testusers.SubscriberProfile{Name: "Bob", Age: 7, Big: math.MaxInt64, Small: math.MaxInt64, Visits: []testusers.Visit{{At: 5}}}}, + {ID: 3, Profile: testusers.SubscriberProfile{Name: "Carol", Age: 61, Visits: []testusers.Visit{}}, Prefs: &testusers.SubscriberPrefs{}, Labels: map[string]string{}}, +} + +func TestASubscribersTypedDocumentsRoundTrip(t *testing.T) { + c := deterministicEQLClient(t) + ctx := context.Background() + encrypted, err := testusers.EncryptSubscriber(ctx, c.DefaultKeyset(), subscribers) + if err != nil { + t.Fatal(err) + } + // A nil pointer is SQL NULL; a pointer to a zero struct is a document. + if encrypted[1].Prefs != nil || encrypted[2].Prefs == nil { + t.Fatalf("prefs: %v %v", encrypted[1].Prefs, encrypted[2].Prefs) + } + if s := encrypted[0].String(); strings.Contains(s, "Alice") { + t.Fatalf("String shows the document: %s", s) + } + back, err := testusers.DecryptSubscriber(ctx, c, encrypted) + if err != nil { + t.Fatal(err) + } + // Every integer to 64 bits and the 17-digit float come back exactly, + // in the declared types; a nil map is a JSON null, and reads back nil. + if !reflect.DeepEqual(back, subscribers) { + t.Fatalf("round trip:\n got %+v\nwant %+v", back, subscribers) + } + // One field's Encrypt writes the same document the record does: it + // opens to the same value through the record's Decrypt. + one, err := testusers.SubscriberFields.Profile.Encrypt(ctx, c.DefaultKeyset(), subscribers[0].Profile) + if err != nil { + t.Fatal(err) + } + rec := encrypted[0] + rec.Profile = one + again, err := testusers.DecryptSubscriber(ctx, c, []testusers.EncryptedSubscriber{rec}) + if err != nil || !reflect.DeepEqual(again[0].Profile, subscribers[0].Profile) { + t.Fatalf("a field's Encrypt: %+v %v", again, err) + } + // A value encoding/json cannot write is refused before anything is + // sealed, naming no value. + bad := []testusers.Subscriber{{ID: 4, Profile: testusers.SubscriberProfile{Score: math.Inf(1)}}} + if _, err := testusers.EncryptSubscriber(ctx, c.DefaultKeyset(), bad); !errors.Is(err, encrypt.ErrEncoding) || strings.Contains(err.Error(), "+Inf") { + t.Fatalf("an infinity: %v", err) + } +} + +func TestASubscribersTypedNeedlesDeriveTheRawQuery(t *testing.T) { + c := deterministicEQLClient(t) + ctx := context.Background() + cipher := c.DefaultKeyset() + f := testusers.SubscriberFields.Profile + // same derives a query twice, once from a Go value and once from the + // JSON text encoding/json writes for it, and wants the same bytes: a + // typed needle is that text, nothing more. + same := func(name string, typed, raw func() ([]byte, error)) { + t.Helper() + a, err := typed() + if err != nil { + t.Fatalf("%s: %v", name, err) + } + b, err := raw() + if err != nil { + t.Fatalf("%s raw: %v", name, err) + } + if !bytes.Equal(a, b) { + t.Errorf("%s: the typed needle derives another query than its JSON text", name) + } + } + bytesOf := func(v []byte, err error) ([]byte, error) { return v, err } + same("contains, a needle struct", + func() ([]byte, error) { + return bytesOf(f.Contains(ctx, cipher, testusers.ProfileNeedle{Tier: ptr("gold"), Visits: []testusers.Visit{{At: 3, Place: "home"}}})) + }, + func() ([]byte, error) { + return bytesOf(f.Contains(ctx, cipher, json.RawMessage(`{"tier":"gold","visits":[{"at":3,"place":"home"}]}`))) + }) + same("contains, a map", + func() ([]byte, error) { return bytesOf(f.Contains(ctx, cipher, map[string]any{"age": 34})) }, + func() ([]byte, error) { return bytesOf(f.Contains(ctx, cipher, json.RawMessage(`{"age":34}`))) }) + same("contained by, the document type", + func() ([]byte, error) { return bytesOf(f.ContainedBy(ctx, cipher, subscribers[1].Profile)) }, + func() ([]byte, error) { + text, err := json.Marshal(subscribers[1].Profile) + if err != nil { + return nil, err + } + return bytesOf(f.ContainedBy(ctx, cipher, json.RawMessage(text))) + }) + same("eq_at, a string", + func() ([]byte, error) { return bytesOf(f.EqAt(ctx, cipher, "$.name", "Alice")) }, + func() ([]byte, error) { return bytesOf(f.EqAt(ctx, cipher, "$.name", json.RawMessage(`"Alice"`))) }) + same("eq_at, a uint64 past 2^53", + func() ([]byte, error) { return bytesOf(f.EqAt(ctx, cipher, "$.big", uint64(math.MaxUint64))) }, + func() ([]byte, error) { + return bytesOf(f.EqAt(ctx, cipher, "$.big", json.RawMessage(`18446744073709551615`))) + }) + // The orderings take Go numbers; a leaf query need not repeat, so each + // is derived and bound, not compared. + if q, err := f.At.Age.Gt(ctx, cipher, 40); err != nil || len(q) == 0 { + t.Fatalf("Gt(40): %v", err) + } + if r, err := f.At.VisitsAt.Between(ctx, cipher, int64(1), 2.5); err != nil || len(r.Low) == 0 || len(r.High) == 0 { + t.Fatalf("Between(1, 2.5): %v", err) + } + + // The field's own struct type is no containment needle: its zero + // fields would narrow the query. Refused before anything is derived, + // by value and by pointer; a type defined over it is the caller's + // choice, and derives. + for name, needle := range map[string]any{ + "the type": testusers.SubscriberProfile{Tier: "gold"}, + "a pointer": &testusers.SubscriberProfile{Tier: "gold"}, + "prefs type": testusers.SubscriberPrefs{Theme: "dark"}, + } { + var err error + if name == "prefs type" { + _, err = testusers.SubscriberFields.Prefs.Contains(ctx, cipher, needle) + } else { + _, err = f.Contains(ctx, cipher, needle) + } + if !errors.Is(err, encrypt.ErrEncoding) || !strings.Contains(err.Error(), "omitempty") || strings.Contains(err.Error(), "gold") { + t.Errorf("%s as a needle: %v", name, err) + } + } + type wholeProfile testusers.SubscriberProfile + if _, err := f.Contains(ctx, cipher, wholeProfile(subscribers[0].Profile)); err != nil { + t.Fatalf("a type defined over the document type: %v", err) + } + // A map of strings is a needle of its own type. + if _, err := testusers.SubscriberFields.Labels.Contains(ctx, cipher, map[string]string{"team": "red"}); err != nil { + t.Fatalf("a map needle: %v", err) + } + // What encoding/json cannot write, and a []byte, are refused, naming + // no value. + for name, err := range map[string]error{ + "[]byte": func() error { _, err := f.Contains(ctx, cipher, []byte(`{"tier":"secret"}`)); return err }(), + "NaN": func() error { _, err := f.At.Age.Lt(ctx, cipher, math.NaN()); return err }(), + "chan": func() error { _, err := f.EqAt(ctx, cipher, "$.name", make(chan int)); return err }(), + } { + if !errors.Is(err, encrypt.ErrEncoding) || strings.Contains(err.Error(), "secret") { + t.Errorf("%s: %v", name, err) + } + } +} diff --git a/languages/golang/encrypt/term.go b/languages/golang/encrypt/term.go index e30d1c97d..0d2251346 100644 --- a/languages/golang/encrypt/term.go +++ b/languages/golang/encrypt/term.go @@ -74,8 +74,9 @@ type JSONOption interface { // which the engine does not derive: a declaration that names it is refused // by stashgen, and at run time by the engine. // -// Deprecated: store a JSON document as a json.RawMessage field with -// `encrypt_into=EncryptedJson`, whose EQL type carries the JSON index. +// Deprecated: store a JSON document as a struct, map, slice or +// json.RawMessage field with `encrypt_into=EncryptedJson`, whose EQL type +// carries the JSON index. func JSON(options ...JSONOption) Index { _ = options return index("json") diff --git a/languages/golang/stashgen/declaration.go b/languages/golang/stashgen/declaration.go index de10e50eb..5be5cb900 100644 --- a/languages/golang/stashgen/declaration.go +++ b/languages/golang/stashgen/declaration.go @@ -278,6 +278,11 @@ const ( // KindJSON is json.RawMessage: a JSON document, stored as an // EncryptedJSON value. KindJSON + // KindJSONValue is a struct, map or slice type stored as an + // EncryptedJSON document: encoding/json writes it to JSON text and reads + // it back. A field is classified so only where it names such an EQL + // type; anywhere else the type keeps its own kind. + KindJSONValue ) var kindNames = [...]string{ @@ -298,6 +303,7 @@ var kindNames = [...]string{ KindTimestamp: "timestamp", KindDecimal: "decimal", KindJSON: "json", + KindJSONValue: "json value", } // String returns the kind's name. @@ -312,7 +318,7 @@ func (k Kind) String() string { func (k Kind) Scalar() bool { switch k { case KindString, KindBool, KindInt, KindUint, KindFloat, KindBytes, - KindInt128, KindUint128, KindDate, KindTimestamp, KindDecimal, KindJSON: + KindInt128, KindUint128, KindDate, KindTimestamp, KindDecimal, KindJSON, KindJSONValue: return true } return false diff --git a/languages/golang/stashgen/emit.go b/languages/golang/stashgen/emit.go index 314781003..ee4b985e8 100644 --- a/languages/golang/stashgen/emit.go +++ b/languages/golang/stashgen/emit.go @@ -590,6 +590,18 @@ func (w *writer) value(f *genFile) { // for it: the struct gets the field's zero value. continue } + if g.GoType.Kind == KindJSONValue { + // A document of a Go type is read back with encoding/json, + // explicitly, into the type the struct declares. + call := "GetDocument" + if g.nullable() { + call = "GetDocumentPtr" + } + w.p("\t\tif v.%s, err = gensupport.%s[%s](vals, %q); err != nil {", g.GoName, call, g.valueExpr, g.Name) + fail() + w.p("\t\t}") + continue + } if g.nullable() && g.definedScalar() { // The engine returns the underlying type, or nil; the // conversion to the defined type is the struct's own. @@ -672,7 +684,11 @@ func (w *writer) fieldEntries(f *genFile) { w.p("}{") for _, g := range sealed { if g.isJSON() { - w.p("\t%s: new%s(gensupport.NewField[%s](%s, %q)),", g.GoName, g.fieldType, g.valueExpr, f.declVar, g.Name) + constructor := "NewField" + if g.holdsStruct { + constructor = "NewDocumentField" + } + w.p("\t%s: new%s(gensupport.%s[%s](%s, %q)),", g.GoName, g.fieldType, constructor, g.valueExpr, f.declVar, g.Name) continue } w.p("\t%s: %s{gensupport.NewField[%s](%s, %q)},", g.GoName, g.fieldType, g.valueExpr, f.declVar, g.Name) @@ -978,9 +994,9 @@ func joinNames(names []string) string { } // isJSON reports whether the field is stored as an EncryptedJSON document: -// a json.RawMessage sealed into an EQL type. +// a json.RawMessage, or a struct, map or slice type, sealed into an EQL type. func (g genField) isJSON() bool { - return g.Verb == VerbEncryptInto && g.GoType.Kind == KindJSON + return g.Verb == VerbEncryptInto && (g.GoType.Kind == KindJSON || g.GoType.Kind == KindJSONValue) } // jsonOperators are the operators a document field answers, and what each @@ -1004,7 +1020,7 @@ func planJSON(f *genFile, g *genField) error { // The engine answers selector for a field whose type stores a json_tree // document; the leaf and entry types below are that family's. if !document { - return fieldErr(f.typeName, g.GoName, "the engine answers no document operator on %s, and the generator writes json.RawMessage fields of EncryptedJson only", g.EQLType) + return fieldErr(f.typeName, g.GoName, "the engine answers no document operator on %s, and the generator writes document fields of EncryptedJson only", g.EQLType) } g.jsonPaths = nil seen := map[string]string{} @@ -1026,7 +1042,6 @@ func planJSON(f *genFile, g *genField) error { g.jsonPaths = append(g.jsonPaths, jsonPath{name: name, path: path}) } } - f.imports.add("encoding/json", "json") return nil } @@ -1104,35 +1119,39 @@ func (w *writer) jsonEntry(g genField) { recv := "func (f " + g.fieldType + ") " if answered["contains"] { w.nl() - w.b.WriteString(wrapComment("Contains derives the query for documents that contain needle, a JSON value: col @> query. Each array element of the needle must sit inside one element of the document's array.", 80)) - w.p("%sContains(ctx context.Context, c *encrypt.Cipher, needle json.RawMessage) (%s, error) {", recv, needle) + doc := "Contains derives the query for documents that contain needle: col @> query. needle is a json.RawMessage of JSON text, or any value encoding/json writes, such as a map or a struct. Each member it writes must be in the document, so a zero-valued struct field narrows the query to documents holding that zero: give a needle struct pointer fields, or omitempty, or use a map. Each array element of the needle must sit inside one element of the document's array." + if g.holdsStruct { + doc += " A needle of the field's own type, " + g.valueExpr + ", is refused for that reason." + } + w.b.WriteString(wrapComment(doc, 80)) + w.p("%sContains(ctx context.Context, c *encrypt.Cipher, needle any) (%s, error) {", recv, needle) w.p("\treturn gensupport.JSONQuery[%s](ctx, f.field, c, \"contains\", needle)", needle) w.p("}") } if answered["contained_by"] { w.nl() - w.b.WriteString(wrapComment("ContainedBy derives the query for documents contained by needle, a JSON value: col <@ query.", 80)) - w.p("%sContainedBy(ctx context.Context, c *encrypt.Cipher, needle json.RawMessage) (%s, error) {", recv, needle) + w.b.WriteString(wrapComment("ContainedBy derives the query for documents contained by needle: col <@ query. needle is a json.RawMessage of JSON text, or any value encoding/json writes; here a zero-valued field widens the query, admitting documents that hold that zero.", 80)) + w.p("%sContainedBy(ctx context.Context, c *encrypt.Cipher, needle any) (%s, error) {", recv, needle) w.p("\treturn gensupport.JSONQuery[%s](ctx, f.field, c, \"contained_by\", needle)", needle) w.p("}") } if answered["eq_at"] { w.nl() - w.b.WriteString(wrapComment("EqAt derives the query for documents holding the scalar v at path: col @> query. A wildcard step matches when any element holds it; a position ([2]) is refused, since the stored selector reads every position as the wildcard.", 80)) - w.p("%sEqAt(ctx context.Context, c *encrypt.Cipher, path string, v json.RawMessage) (%s, error) {", recv, needle) + w.b.WriteString(wrapComment("EqAt derives the query for documents holding the scalar v at path: col @> query. v is a Go string, number or bool, nil for JSON null, or a json.RawMessage of JSON text. A wildcard step matches when any element holds it; a position ([2]) is refused, since the stored selector reads every position as the wildcard.", 80)) + w.p("%sEqAt(ctx context.Context, c *encrypt.Cipher, path string, v any) (%s, error) {", recv, needle) w.p("\treturn gensupport.JSONQueryAt[%s](ctx, f.field, c, \"eq_at\", path, v)", needle) w.p("}") } if answered["selector"] { w.nl() - w.b.WriteString(wrapComment("Selector prepares `->` at path: bind Selector() as col -> $1::text, and open the entries it returns with Open. A position ([2]) is refused; read one with Subtree.", 80)) + w.b.WriteString(wrapComment("Selector prepares `->` at path: bind Selector() as col -> $1::text, and open the entries it returns with Open; encrypt.DecodeJSON reads each value into a Go type. A position ([2]) is refused; read one with Subtree.", 80)) w.p("%sSelector(ctx context.Context, c *encrypt.Cipher, path string) (encrypt.JSONSelection[%s], error) {", recv, entry) w.p("\treturn gensupport.JSONSelect[%s](ctx, f.field, c, path)", entry) w.p("}") } if answered["subtree"] { w.nl() - w.b.WriteString(wrapComment("Subtree prepares a read of the value at path: bind Tokens() as eql_v4.json_tree_subtree(col, $1::text[], budget), and open the rows it returns with Open or OpenOne.", 80)) + w.b.WriteString(wrapComment("Subtree prepares a read of the value at path: bind Tokens() as eql_v4.json_tree_subtree(col, $1::text[], budget), and open the rows it returns with Open or OpenOne; encrypt.DecodeJSON reads each value into a Go type.", 80)) w.p("%sSubtree(ctx context.Context, c *encrypt.Cipher, path string, opts encrypt.JSONReadOptions) (encrypt.JSONRead[%s], error) {", recv, entry) w.p("\treturn gensupport.JSONSubtree[%s](ctx, f.field, c, path, opts)", entry) w.p("}") @@ -1161,7 +1180,7 @@ func (w *writer) jsonEntry(g genField) { } w.p("}") w.nl() - w.b.WriteString(wrapComment(fmt.Sprintf("%sPath is one ordered path of %s: its orderings compare the first value `->` extracts there, by the leaf's ordering term. Numbers order as float64, so integers past 2^53 can tie; text orders case- and accent-folded.", g.fieldType, g.GoName), 80)) + w.b.WriteString(wrapComment(fmt.Sprintf("%sPath is one ordered path of %s: its orderings compare the first value `->` extracts there, by the leaf's ordering term. Each operand is a Go number or string, or a json.RawMessage of JSON text. Numbers order as float64, so integers past 2^53 can tie; text orders case- and accent-folded.", g.fieldType, g.GoName), 80)) w.p("type %sPath struct {", g.fieldType) w.p("\tfield %s", field) w.p("\tpath string") @@ -1171,13 +1190,13 @@ func (w *writer) jsonEntry(g genField) { continue } w.nl() - w.p("func (p %sPath) %s(ctx context.Context, c *encrypt.Cipher, v json.RawMessage) (%s, error) {", g.fieldType, op.method, leaf) + w.p("func (p %sPath) %s(ctx context.Context, c *encrypt.Cipher, v any) (%s, error) {", g.fieldType, op.method, leaf) w.p("\treturn gensupport.JSONQueryAt[%s](ctx, p.field, c, %q, p.path, v)", leaf, op.name) w.p("}") } if answered["between_at"] { w.nl() - w.p("func (p %sPath) Between(ctx context.Context, c *encrypt.Cipher, lo, hi json.RawMessage) (encrypt.JSONRange[%s], error) {", g.fieldType, leaf) + w.p("func (p %sPath) Between(ctx context.Context, c *encrypt.Cipher, lo, hi any) (encrypt.JSONRange[%s], error) {", g.fieldType, leaf) w.p("\treturn gensupport.JSONBetweenAt[%s](ctx, p.field, c, p.path, lo, hi)", leaf) w.p("}") } diff --git a/languages/golang/stashgen/engine.go b/languages/golang/stashgen/engine.go index 08275aaa1..ff9a2e820 100644 --- a/languages/golang/stashgen/engine.go +++ b/languages/golang/stashgen/engine.go @@ -225,10 +225,13 @@ func lowerDeclaration(d Declaration, eqlTypes []EQLType) (*record.Plan, error) { // intoKind takes every built-in integer except into a // 128-bit kind, which gensupport does not convert to. return nil, &FieldError{Type: d.Type, Field: f.GoName, Reason: fmt.Sprintf("%s seals %s, and the built-in integer %s is not converted to it", f.EQLType, eqlType.wire, f.GoType)} - } else if eqlType.wire == record.JSON && f.GoType.Kind != KindJSON { + } else if eqlType.wire == record.JSON { // A document's kind is no Go scalar's: name the Go types - // that carry one, not a kind mismatch. - return nil, &FieldError{Type: d.Type, Field: f.GoName, Reason: fmt.Sprintf("%s seals a json document: declare the field json.RawMessage, or *json.RawMessage with null=sql", f.EQLType)} + // that carry one, not a kind mismatch. A json.RawMessage + // and a type encoding/json carries both cross as JSON text. + if f.GoType.Kind != KindJSON && f.GoType.Kind != KindJSONValue { + return nil, &FieldError{Type: d.Type, Field: f.GoName, Reason: fmt.Sprintf("%s seals a json document: %s", f.EQLType, documentTypes)} + } } else { // The engine checks the kind too (se_plan_check refuses a // "type" other than the EQL type's plaintext); saying it @@ -271,7 +274,7 @@ func lowerDeclaration(d Declaration, eqlTypes []EQLType) (*record.Plan, error) { if !ok { reason := fmt.Sprintf("the engine does not derive the %s index yet", idx.Name) if idx.Name == IndexJSON { - reason = "the json index is not an output beside a ciphertext: store a JSON document as a json.RawMessage field with encrypt_into=EncryptedJson, whose EQL type carries the index" + reason = "the json index is not an output beside a ciphertext: store a JSON document as a struct, map, slice or json.RawMessage field with encrypt_into=EncryptedJson, whose EQL type carries the index" } return nil, &FieldError{Type: d.Type, Field: f.GoName, Reason: reason} } @@ -339,12 +342,15 @@ func wireKind(t GoType) record.Kind { return record.Timestamp case KindDecimal: return record.Decimal - case KindJSON: + case KindJSON, KindJSONValue: return record.JSON } return record.Untyped } +// documentTypes names the Go types a field stored as a JSON document takes. +const documentTypes = "declare the field json.RawMessage, or a struct, map or slice type that encoding/json writes and reads (a pointer to either with null=sql)" + // kindOfWire is the generator's kind for a wire kind, for an EQL type's // plaintext. func kindOfWire(k record.Kind) Kind { diff --git a/languages/golang/stashgen/read.go b/languages/golang/stashgen/read.go index 2a98fe518..d7edfad3c 100644 --- a/languages/golang/stashgen/read.go +++ b/languages/golang/stashgen/read.go @@ -3,6 +3,7 @@ package stashgen import ( "fmt" "go/types" + "reflect" "strings" "unicode" "unicode/utf8" @@ -107,6 +108,9 @@ type genField struct { fieldType string // "EmailField" pathType string // the type with its package path, for model checks jsonPaths []jsonPath // a document field's ordered paths, each with its accessor + // holdsStruct is set for a document field (KindJSONValue) whose type + // holds a struct: its Contains refuses a needle of that type. + holdsStruct bool } // definedScalar reports whether the field's type is a named type over a @@ -714,6 +718,25 @@ func (r *reader) buildFields(c *collected) error { if t.Omit { field.Verb = VerbOmit } + // A field that names an EQL type storing a JSON document, of a Go + // type other than json.RawMessage, is a document that encoding/json + // writes and reads: a struct, map or slice, or a pointer to one. + // What encoding/json cannot carry both ways is refused here, when + // the code is generated. + holdsStruct := false + if field.Verb == VerbEncryptInto && r.storesDocument(field.EQLType) { + doc, docType := &field.GoType, cf.typ + if doc.Kind == KindPointer && doc.Elem != nil { + doc, docType = doc.Elem, docType.Underlying().(*types.Pointer).Elem() + } + if doc.Kind != KindJSON { + if reason := notDocument(docType); reason != "" { + return fieldErr(typeName, cf.goName, "%s seals a json document, and %s %s; %s", field.EQLType, r.typeExpr(docType), reason, documentTypes) + } + holdsStruct = reaches(docType, func(u types.Type) bool { _, ok := u.(*types.Struct); return ok }, map[types.Type]bool{}) + *doc = GoType{Name: doc.Name, Kind: KindJSONValue} + } + } // A json.RawMessage is a document only where it names an EQL type // that stores one; sealed or indexed any other way it is bytes, as // it was before documents were. @@ -748,7 +771,7 @@ func (r *reader) buildFields(c *collected) error { if t.Omit { continue } - g := genField{Field: field, typeExpr: r.typeExpr(cf.typ), via: cf.via, tags: cf.tags, pathType: pathType(cf.typ)} + g := genField{Field: field, typeExpr: r.typeExpr(cf.typ), via: cf.via, tags: cf.tags, pathType: pathType(cf.typ), holdsStruct: holdsStruct} g.valueExpr = g.typeExpr if pointer { g.valueExpr = field.GoType.Name @@ -756,7 +779,7 @@ func (r *reader) buildFields(c *collected) error { if g.Sealed() && !g.GoType.Kind.Scalar() { // The engine seals a composite as a tree of leaves, and a column // holds one leaf; only an opaque struct seals a whole value. - return fieldErr(typeName, cf.goName, "a sealed field is one scalar (a string, number, bool, []byte, time.Time, encrypt.Date, encrypt.Decimal, encrypt.Int128 or encrypt.Uint128, or a type defined over a string, number, bool or []byte); a %s seals only as part of an opaque struct", g.typeExpr) + return fieldErr(typeName, cf.goName, "a sealed field is one scalar (a string, number, bool, []byte, time.Time, encrypt.Date, encrypt.Decimal, encrypt.Int128 or encrypt.Uint128, or a type defined over a string, number, bool or []byte); a %s seals only as part of an opaque struct, or as a JSON document with encrypt_into=EncryptedJson", g.typeExpr) } switch field.Verb { case VerbPassthrough, VerbContextField: @@ -1142,6 +1165,117 @@ func notJSONEncodable(t types.Type, seen map[types.Type]bool) string { return "is not a type JSON carries" } +// storesDocument reports whether the engine's EQL type of that name stores +// a JSON document. +func (r *reader) storesDocument(name string) bool { + for _, t := range r.eql { + if t.Name == name { + return t.wire == record.JSON + } + } + return false +} + +// notDocument says why a Go type other than json.RawMessage is not a JSON +// document that encoding/json writes and reads back to the same value, or +// "" when it is: a struct, map, slice or array whose every part +// encoding/json carries ([notJSONEncodable]) and holds no interface. +func notDocument(t types.Type) string { + if _, ok := valueKind(t); ok { + return "is one value, not a document" + } + switch u := t.Underlying().(type) { + case *types.Basic: + return "is one JSON scalar, not a document (a string is a JSON string to encoding/json, not the JSON text it may hold)" + case *types.Slice: + if b, ok := u.Elem().Underlying().(*types.Basic); ok && b.Kind() == types.Byte { + return "is written by encoding/json as a base64 string, not as JSON text" + } + case *types.Pointer: + return "is a pointer to a pointer" + case *types.Struct, *types.Map, *types.Array: + default: + return "is not a type JSON carries" + } + if reaches(t, func(u types.Type) bool { _, ok := u.(*types.Interface); return ok }, map[types.Type]bool{}) { + return "holds an interface, which encoding/json reads back as a map or a float64, not the value written" + } + if name := droppedField(t, map[types.Type]bool{}); name != "" { + return "has a field " + name + ` tagged json:"-", which encoding/json drops, so the stored document would not hold it` + } + return notJSONEncodable(t, map[types.Type]bool{}) +} + +// droppedField names the first exported struct field that t, or a type it +// reaches, tags json:"-", or "" for none: encoding/json writes no such +// field, so a document of the type would come back without it. A type that +// writes and reads itself is not looked into. +func droppedField(t types.Type, seen map[types.Type]bool) string { + if seen[t] { + return "" + } + seen[t] = true + if hasMethod(types.NewPointer(t), "UnmarshalJSON") && hasMethod(t, "MarshalJSON") { + return "" + } + switch u := t.Underlying().(type) { + case *types.Slice: + return droppedField(u.Elem(), seen) + case *types.Array: + return droppedField(u.Elem(), seen) + case *types.Pointer: + return droppedField(u.Elem(), seen) + case *types.Map: + return droppedField(u.Elem(), seen) + case *types.Struct: + for i := range u.NumFields() { + fld := u.Field(i) + if fld.Exported() && reflect.StructTag(u.Tag(i)).Get("json") == "-" { + return fld.Name() + } + if name := droppedField(fld.Type(), seen); name != "" { + return name + } + } + } + return "" +} + +// reaches reports whether t, or a type its fields, elements or pointers +// reach, has an underlying type that match reports: a document type that +// holds a struct, say. A type that writes and reads itself (MarshalJSON and +// UnmarshalJSON) is not looked into, as encoding/json does not. +func reaches(t types.Type, match func(types.Type) bool, seen map[types.Type]bool) bool { + if seen[t] { + return false + } + seen[t] = true + if hasMethod(types.NewPointer(t), "UnmarshalJSON") && hasMethod(t, "MarshalJSON") { + return match(t.Underlying()) + } + u := t.Underlying() + if match(u) { + return true + } + switch u := u.(type) { + case *types.Slice: + return reaches(u.Elem(), match, seen) + case *types.Array: + return reaches(u.Elem(), match, seen) + case *types.Pointer: + return reaches(u.Elem(), match, seen) + case *types.Map: + return reaches(u.Elem(), match, seen) + case *types.Struct: + for i := range u.NumFields() { + if reaches(u.Field(i).Type(), match, seen) { + return true + } + } + } + return false +} + // notStorable says why the storage profile (vcffi.MarshalStorage and // UnmarshalStorage) cannot carry a type both ways, or "" when it can, by // vcffi's own rules: a scalar by its kind; a type with MarshalText and diff --git a/languages/golang/stashgen/testdata/stubsdk/encrypt/encrypt.go b/languages/golang/stashgen/testdata/stubsdk/encrypt/encrypt.go index 6ddfbe7bc..0482e394f 100644 --- a/languages/golang/stashgen/testdata/stubsdk/encrypt/encrypt.go +++ b/languages/golang/stashgen/testdata/stubsdk/encrypt/encrypt.go @@ -8,6 +8,7 @@ package encrypt import ( "context" + "encoding/json" "time" ) @@ -135,6 +136,9 @@ type JSONValue struct { JSON []byte } +// DecodeJSON reads JSON text a read opened into a value of type T. +func DecodeJSON[T any](json.RawMessage) (T, error) { var z T; return z, nil } + // JSONReadOptions are a subtree read's options. type JSONReadOptions struct{ WildcardBound uint32 } diff --git a/languages/golang/stashgen/testdata/stubsdk/encrypt/gensupport/gensupport.go b/languages/golang/stashgen/testdata/stubsdk/encrypt/gensupport/gensupport.go index 614303dae..d382e3334 100644 --- a/languages/golang/stashgen/testdata/stubsdk/encrypt/gensupport/gensupport.go +++ b/languages/golang/stashgen/testdata/stubsdk/encrypt/gensupport/gensupport.go @@ -7,7 +7,6 @@ package gensupport import ( "context" - "encoding/json" "log/slog" "github.com/cipherstash/stack/languages/golang/encrypt" @@ -131,6 +130,10 @@ type Field[T any] struct{ _ struct{} } // NewField makes the entry for one field of a declaration. func NewField[T any](Declaration, string) Field[T] { return Field[T]{} } +// NewDocumentField makes the entry for a document field whose type holds a +// struct. +func NewDocumentField[T any](Declaration, string) Field[T] { return Field[T]{} } + func (Field[T]) Encrypt(context.Context, *encrypt.Cipher, T) (Output, error) { return Output{}, nil } func (Field[T]) Term(context.Context, *encrypt.Cipher, string, T) ([]byte, error) { return nil, nil } func (Field[T]) Range(context.Context, *encrypt.Cipher, string, T, T) ([]byte, []byte, error) { @@ -200,6 +203,15 @@ func (f Field[T]) NullTerm(context.Context, *encrypt.Cipher, string) ([]byte, er // GetPtr reads one opened field declared over a pointer. func GetPtr[T any](Values, string) (*T, error) { return nil, nil } +// GetDocument reads one opened document field into its Go type. +func GetDocument[T any](Values, string) (T, error) { var z T; return z, nil } + +// GetDocumentPtr reads one opened document field declared over a pointer. +func GetDocumentPtr[T any](Values, string) (*T, error) { return nil, nil } + +// JSONText is a Go value as JSON text. +func JSONText(any) ([]byte, error) { return nil, nil } + // TargetOptions are the options an EncryptInto field gives its EQL type. type TargetOptions struct { SealedRoot *bool @@ -210,17 +222,17 @@ type TargetOptions struct { func Ptr[T any](v T) *T { return &v } // JSONQuery derives a containment operator's query on a JSON field. -func JSONQuery[Q ~[]byte, T any](context.Context, Field[T], *encrypt.Cipher, string, json.RawMessage) (Q, error) { +func JSONQuery[Q ~[]byte, T any](context.Context, Field[T], *encrypt.Cipher, string, any) (Q, error) { return nil, nil } // JSONQueryAt derives an operator at a path on a JSON field. -func JSONQueryAt[Q ~[]byte, T any](context.Context, Field[T], *encrypt.Cipher, string, string, json.RawMessage) (Q, error) { +func JSONQueryAt[Q ~[]byte, T any](context.Context, Field[T], *encrypt.Cipher, string, string, any) (Q, error) { return nil, nil } // JSONBetweenAt derives between_at on a JSON field. -func JSONBetweenAt[Q ~[]byte, T any](context.Context, Field[T], *encrypt.Cipher, string, json.RawMessage, json.RawMessage) (encrypt.JSONRange[Q], error) { +func JSONBetweenAt[Q ~[]byte, T any](context.Context, Field[T], *encrypt.Cipher, string, any, any) (encrypt.JSONRange[Q], error) { return encrypt.JSONRange[Q]{}, nil } diff --git a/mise.toml b/mise.toml index 930072ba3..a8ca38e6b 100644 --- a/mise.toml +++ b/mise.toml @@ -368,7 +368,7 @@ description = "Format check, vet and test the Go module (languages/golang: encry run = "scripts/go-binding-test.sh languages/golang" [tasks."go:test:postgres"] -description = "Install the EQL v4 bundle built from the tree into the database EQL_TEST_DATABASE_URL names and run the Go module's PostgreSQL tests against it: rows the Go SDK stores in every EQL type it produces, found by the queries it derives, encrypted JSON documents queried and read through their generated methods, and random values in every ordered type, ordered as their plaintexts (EQL_ORDER_SEED replays a seed); needs `wasm:guest:build:deterministic` and `wasm:guest:build:eql:deterministic` first" +description = "Install the EQL v4 bundle built from the tree into the database EQL_TEST_DATABASE_URL names and run the Go module's PostgreSQL tests against it: rows the Go SDK stores in every EQL type it produces, found by the queries it derives, encrypted JSON documents, raw and of Go types, queried and read through their generated methods, and random values in every ordered type, ordered as their plaintexts (EQL_ORDER_SEED replays a seed); needs `wasm:guest:build:deterministic` and `wasm:guest:build:eql:deterministic` first" # The bundle is built from packages/eql, as test:encryption:postgres builds # it: the copy under crates/eql-bindings/sql is the RELEASED bundle and moves # only at the lockstep bump, so it cannot show a SQL change in the same pull @@ -384,7 +384,7 @@ cd languages/golang # The tests skip without DATABASE_URL and without their guest builds; -v and # the grep turn either skip into a failure here, where both are provided. DATABASE_URL="$EQL_TEST_DATABASE_URL" CGO_ENABLED=0 go test -count=1 -v \ - -run '^(TestEQLTypesRoundTripAndQueryInPostgres|TestEQLOrderMatchesPlaintextInPostgres|TestQueriesRunAgainstPostgres|TestJSONDocumentsQueryAndReadInPostgres)$' ./encrypt \ + -run '^(TestEQLTypesRoundTripAndQueryInPostgres|TestEQLOrderMatchesPlaintextInPostgres|TestQueriesRunAgainstPostgres|TestJSONDocumentsQueryAndReadInPostgres|TestJSONTypedDocumentsInPostgres)$' ./encrypt \ | tee "${TMPDIR:-/tmp}/go-test-postgres.log" if grep -q -- '--- SKIP' "${TMPDIR:-/tmp}/go-test-postgres.log"; then echo "error: a PostgreSQL test skipped; build the deterministic guests first" >&2 @@ -392,7 +392,7 @@ if grep -q -- '--- SKIP' "${TMPDIR:-/tmp}/go-test-postgres.log"; then fi # A -run pattern that matches no test (one renamed, say) prints "no tests # to run" and PASS, so each test has to be seen to pass by name. -for test in TestEQLTypesRoundTripAndQueryInPostgres TestEQLOrderMatchesPlaintextInPostgres TestQueriesRunAgainstPostgres TestJSONDocumentsQueryAndReadInPostgres; do +for test in TestEQLTypesRoundTripAndQueryInPostgres TestEQLOrderMatchesPlaintextInPostgres TestQueriesRunAgainstPostgres TestJSONDocumentsQueryAndReadInPostgres TestJSONTypedDocumentsInPostgres; do if ! grep -q -- "^--- PASS: $test " "${TMPDIR:-/tmp}/go-test-postgres.log"; then echo "error: $test did not run and pass; was it renamed?" >&2 exit 1 From d2d236c5441e7271d4d48b823402a892cfd6e6bc Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 12:10:57 +1100 Subject: [PATCH 02/12] docs(go): document encrypted JSON documents of Go types The stashgen and encrypt READMEs, the encrypt package docs and the CHANGELOG describe struct, map and slice documents, Go-valued operands, encrypt.DecodeJSON, and the containment narrowing rule: a zero struct field without omitempty narrows the query, so a needle type uses pointer fields with omitempty, or a map. Refs #1351. Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- languages/golang/CHANGELOG.md | 20 ++++++++++ languages/golang/cmd/stashgen/README.md | 53 ++++++++++++++++++++++--- languages/golang/encrypt/README.md | 22 +++++++--- languages/golang/encrypt/doc.go | 27 +++++++++++++ 4 files changed, 111 insertions(+), 11 deletions(-) diff --git a/languages/golang/CHANGELOG.md b/languages/golang/CHANGELOG.md index 5fad839ff..9e16626d3 100644 --- a/languages/golang/CHANGELOG.md +++ b/languages/golang/CHANGELOG.md @@ -12,6 +12,26 @@ listed under **Breaking**. Releases are tagged `languages/golang/vX.Y.Z`. ### Added +- **Encrypted JSON documents of Go types** (#1351). A struct, map or slice + field tagged `encrypt_into=EncryptedJson` is a document: the generated code + marshals it with `encoding/json` when it seals it, and reads the opened + document back into the declared type (`gensupport.GetDocument`, + `GetDocumentPtr`), so 64-bit integers and finite `float64`s round-trip + exactly. A pointer to one takes `null=sql`. `stashgen` refuses, when it + runs, a type `encoding/json` cannot carry both ways: an interface such as + `any`, an unexported struct field or one tagged `json:"-"`, a channel, a + function, a scalar, or a `[]byte`. The `Fields` entry's operands are Go values now: `Contains`, + `ContainedBy`, `EqAt`, `Lt`, `Lte`, `Gt`, `Gte` and `Between` take `any`, + a `json.RawMessage` crossing as the JSON text it is and anything else + marshalled (`gensupport.JSONText`); a plain `[]byte` is refused. A zero + struct field without `omitempty` narrows a containment query, so + `Contains` on a field whose type holds a struct refuses a needle of that + type (`gensupport.NewDocumentField`); the generated doc comments say how + to write a needle. `encrypt.DecodeJSON[T]` reads a value a selection or + subtree read opened into a Go type, numbers in an interface as + `json.Number`. An error from either direction names a Go type and never + the value. + - **Encrypted JSON documents with `stashgen`** (#1351, #1352). A `json.RawMessage` field tagged `encrypt_into=EncryptedJson` (options: `ordered=$.a;$.b[*].c`, `sealed_root=false`) is stored as an diff --git a/languages/golang/cmd/stashgen/README.md b/languages/golang/cmd/stashgen/README.md index 17ca31938..e35bde131 100644 --- a/languages/golang/cmd/stashgen/README.md +++ b/languages/golang/cmd/stashgen/README.md @@ -74,7 +74,7 @@ The first part of a tag is the field's name, which is the column name in a datab | `stash:"notes,encrypt"` | seal the field, with no index | | `stash:"email,encrypt,index=equality;match"` | seal the field, and derive each index beside it | | `stash:"score,index=ore"` | derive the index alone; no ciphertext is stored, so `Decrypt` leaves the field at its zero value | -| `stash:"profile,encrypt_into=EncryptedJson(ordered=$.age)"` | on a `json.RawMessage`: store the field as an encrypted JSON document; see "Encrypted JSON documents" | +| `stash:"profile,encrypt_into=EncryptedJson(ordered=$.age)"` | on a `json.RawMessage`, or a struct, map or slice type: store the field as an encrypted JSON document; see "Encrypted JSON documents" | | `stash:"attrs,index=json"` | refused: a JSON document is stored with `encrypt_into=EncryptedJson`, not as an index beside a ciphertext | | `stash:"id,passthrough"` | store the field as it is | | `stash:"tenant,context_field"` | the field's value is the context of every other field in the struct, in place of a `context=` tag; stored as it is | @@ -113,10 +113,43 @@ These words are the same as the Rust API's words for the same behaviour, with tw ### Encrypted JSON documents -A `json.RawMessage` field with `encrypt_into=EncryptedJson` is stored as one encrypted JSON document, an `eql.EncryptedJSON` in a `public.eql_v4_json_tree` column. -The field holds the document's JSON text; the engine parses it, so an object that names a member twice, an integer outside the 64-bit range and text that is not JSON are refused with `encrypt.ErrEncoding`. -Values come back as JSON text holding the same values; key order and the spelling of numbers are not kept. -A `*json.RawMessage` field is optional and must say `null=sql`: a document has no fixed length to hide a missing one behind. A non-nil pointer to `null` is a stored JSON `null`. +A field with `encrypt_into=EncryptedJson` is stored as one encrypted JSON document, an `eql.EncryptedJSON` in a `public.eql_v4_json_tree` column. +Declare it as a struct, map or slice type, or as a `json.RawMessage` that holds the document's JSON text: + +```go +type Profile struct { + Name string `json:"name"` + Age int64 `json:"age"` + Visits []Visit `json:"visits"` +} + +type Customer struct { + _ struct{} `stash:"context=customers"` + ID int64 `stash:"id,passthrough"` + Profile Profile `stash:"profile,encrypt_into=EncryptedJson(ordered=$.age)"` + Notes json.RawMessage `stash:"notes,encrypt_into=EncryptedJson"` +} + +// A needle for Contains: a field left nil is not in the query. +type ProfileNeedle struct { + Name *string `json:"name,omitempty"` + Age *int64 `json:"age,omitempty"` +} + +q, err := customers.Fields.Profile.Contains(ctx, cipher, ProfileNeedle{Age: &age}) +older, err := customers.Fields.Profile.At.Age.Gt(ctx, cipher, 40) +people, err := customers.Decrypt(ctx, cipher, rows) // each Profile is a Profile again +``` + +The generated code marshals a struct, map or slice type with `encoding/json` when it seals the field, and unmarshals the opened document into the declared type. +It calls `encoding/json` directly, never through `map[string]any`, so an `int64`, a `uint64` and a finite `float64` come back exactly. +`stashgen` refuses a type that `encoding/json` cannot carry both ways: an interface such as `any` anywhere in it (it reads numbers back as `float64`), an unexported struct field or one tagged `json:"-"` (each is dropped), a channel or a function. +It refuses a string, a number and a bool too: a document is a struct, map or slice. +A `[]byte` is refused, since `encoding/json` writes it as base64: declare JSON text as `json.RawMessage`. +The engine parses the JSON text, so an object that names a member twice and an integer outside the 64-bit range are refused with `encrypt.ErrEncoding`. +A `json.RawMessage` comes back as JSON text that holds the same values; key order and the spelling of numbers are not kept. +A pointer field (`*Profile`, `*json.RawMessage`) is optional and must say `null=sql`: a document has no fixed length to hide a missing one behind. +A nil pointer is SQL `NULL`; a non-nil `*json.RawMessage` holding `null` is a stored JSON `null`. A `json.RawMessage` sealed any other way (`encrypt`, `index=equality`) is bytes, as any type defined over `[]byte` is. The type takes options in parentheses: `ordered=` names the paths whose numbers and strings carry an ordering term, separated by `;`, and `sealed_root=false` turns off the sealed root (on by default). @@ -137,7 +170,15 @@ The field's `Fields` entry has a method per operation: `At` has one accessor per ordered path, named for its keys (`$.visits[*].at` is `At.VisitsAt`, `$.a.1` is `At.A1`); two paths with one name, or one path given twice, stop `stashgen`. An ordering compares the value `->` extracts at its path, and EQL's `<` compares ordering terms alone, so take the selector from the same accessor (`At.Age.Selector`) rather than spelling the path again. `OpenOne` reads one value and refuses a read whose path has a wildcard before anything is opened; open a wildcard read with `Open`. -The needle and the values are `json.RawMessage`: write the JSON you mean, such as `json.RawMessage(`+"`"+`{"active":true}`+"`"+`)`. +A needle and a value are Go values: a `json.RawMessage` of JSON text crosses as it is, and anything else is marshalled with `encoding/json`. +`EqAt` and the orderings take a scalar: a string, a number, a bool, or `nil` for JSON `null`. +A plain `[]byte` is refused rather than written as base64. +A document contains a needle when it holds every member the needle writes, and `encoding/json` writes a struct field that has no `omitempty` even when it is zero. +So a zero field narrows the query to the documents that hold that zero: `Profile{Name: "Alice"}` also asks for `"age": 0`. +Give a needle struct pointer fields with `omitempty`, as `ProfileNeedle` above, or use a map. +`Contains` refuses a needle of the field's own type when that type holds a struct, since that is the likeliest way to make the mistake; a type defined over it (`type wholeProfile Profile`) states that you mean every field. +`ContainedBy` takes the field's own type: there a zero field widens the query. +Each value a selection or read opens is JSON text; `encrypt.DecodeJSON[T]` reads one into a Go type, with the same rules as a whole document. A position in a path is refused for `EqAt` and `Selector`, whose selector reads every position as the wildcard; read a position with `Subtree`. ### Optional fields diff --git a/languages/golang/encrypt/README.md b/languages/golang/encrypt/README.md index 5ba5516a5..302d81e58 100644 --- a/languages/golang/encrypt/README.md +++ b/languages/golang/encrypt/README.md @@ -594,21 +594,33 @@ column. ### Encrypted JSON documents `EncryptedJson` stores a whole JSON document in one `public.eql_v4_json_tree` -column, from a `json.RawMessage` field (`stash:"profile,encrypt_into=EncryptedJson(ordered=$.age)"`; -see the stashgen README). One entry per node is encrypted, so the database -can test containment, equality and ordering at a path, return one node with -`->`, and walk a subtree, without decrypting: +column, from a struct, map or slice field, or a `json.RawMessage` +(`stash:"profile,encrypt_into=EncryptedJson(ordered=$.age)"`; see the +stashgen README). A struct is marshalled with `encoding/json` when it is +sealed and read back into its type when it is opened. One entry per node is +encrypted, so the database can test containment, equality and ordering at a +path, return one node with `->`, and walk a subtree, without decrypting: ```go -q, err := users.Fields.Profile.Contains(ctx, cipher, json.RawMessage(`{"plan":"pro"}`)) +q, err := users.Fields.Profile.Contains(ctx, cipher, map[string]any{"plan": "pro"}) rows, err := db.QueryContext(ctx, `SELECT id FROM users WHERE profile @> $1::eql_v4.query_json_tree`, q) read, err := users.Fields.Profile.Subtree(ctx, cipher, "$.visits[*].at", encrypt.JSONReadOptions{}) // SELECT e FROM users, eql_v4.json_tree_subtree(profile, $1::text[], 64) AS e WHERE id = $2 // with read.Tokens() as $1; collect each row's entries, then: values, err := read.Open(ctx, entriesPerRow) +at, err := encrypt.DecodeJSON[int64](values[0][0]) ``` +A needle or a value is a `json.RawMessage` of JSON text, or any Go value +`encoding/json` writes. A document contains a needle when it holds every +member the needle writes, and `encoding/json` writes a zero struct field +that has no `omitempty`: that field narrows the query to documents holding +the zero. Give a needle struct pointer fields with `omitempty`, or use a +map. `Contains` refuses a needle of the field's own struct type for this +reason. Each value a read opens is JSON text, and `encrypt.DecodeJSON` reads +it into a Go type. + A selection or read opens rows only as answers to the request that prepared it: the engine derives the selector or the read again from the field, the path and the cipher's keyset, and refuses a row that is not one it asked diff --git a/languages/golang/encrypt/doc.go b/languages/golang/encrypt/doc.go index 2f9c1e509..4fb6d22e0 100644 --- a/languages/golang/encrypt/doc.go +++ b/languages/golang/encrypt/doc.go @@ -200,6 +200,33 @@ // memory until the garbage collector reclaims it; a MatchQuery never prints // it. // +// # Encrypted JSON documents +// +// A field tagged encrypt_into=EncryptedJson is stored as one encrypted JSON +// document. Declare it as a json.RawMessage, which holds the document's JSON +// text, or as a struct, map or slice type. The generated code marshals such +// a type with encoding/json when it seals the field. It unmarshals the +// opened text into the declared type, so integers to 64 bits and finite +// float64s come back exactly. +// +// The field's operands are Go values. Contains and ContainedBy take a +// needle, and EqAt and the orderings take a scalar. Each is a +// json.RawMessage of JSON text or any value encoding/json writes: +// +// q, err := customers.Fields.Profile.Contains(ctx, cipher, map[string]any{"tier": "gold"}) +// older, err := customers.Fields.Profile.At.Age.Gt(ctx, cipher, 40) +// +// A document contains a needle when it holds every member the needle writes. +// encoding/json writes a struct field that has no omitempty even when the +// field is zero. So a zero field narrows the query to documents that hold +// that zero. Give a needle struct pointer fields with omitempty, or use a +// map. Contains refuses a needle of the field's own type when that type +// holds a struct, the likeliest way to make this mistake. ContainedBy takes +// one: there a zero field widens the query. +// +// A selection or a subtree read opens values as JSON text ([JSONValue], +// [JSONRead.Open]). [DecodeJSON] reads one into a Go type. +// // # Cancellation // // Every call takes a [context.Context], and a call whose context is From 586e27c53a781aaa6dd1c8c9dfb88c78c80ffb4f Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 12:49:11 +1100 Subject: [PATCH 03/12] fix(go): close the typed JSON document review findings stashgen's document check is one walk (documentWalk) in place of three near-copies, and it now refuses what encoding/json would write one way and read another: a MarshalJSON or MarshalText with no matching unmarshal method, a marshal method on the pointer alone (not called for a value), a codec a struct takes from a field it embeds (it writes that field and drops the rest), and two fields that write one JSON member (encoding/json keeps one, or neither, silently). An unexported embedded struct is accepted: encoding/json promotes its fields. One eqlType lookup serves buildFields, nullMode and the document check. JSONText refuses any type defined over []byte, not only []byte itself: type Doc []byte, or json.RawMessage redefined, has no MarshalJSON and would be written as base64. Contains and ContainedBy refuse a Go string needle: it would be one JSON string, never the JSON text it holds, and the operands' move to any had made that mistake compile. A string stays the value for EqAt and the orderings, and the generated comments say so. DecodeJSON tells trailing text that is not JSON from a second value, and names an interface type T, which %T of a nil printed as . Refs #1351. Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- languages/golang/CHANGELOG.md | 8 +- languages/golang/cmd/stashgen/README.md | 7 +- languages/golang/cmd/stashgen/main_test.go | 18 +- languages/golang/encrypt/decode_json_test.go | 8 + languages/golang/encrypt/doc.go | 3 + languages/golang/encrypt/gensupport/json.go | 46 +++- .../internal/testusers/customer_stash.go | 48 ++-- .../internal/testusers/subscriber_stash.go | 77 +++--- languages/golang/encrypt/json.go | 29 ++- languages/golang/encrypt/json_typed_test.go | 18 +- languages/golang/stashgen/emit.go | 6 +- languages/golang/stashgen/read.go | 230 +++++++++++------- 12 files changed, 326 insertions(+), 172 deletions(-) diff --git a/languages/golang/CHANGELOG.md b/languages/golang/CHANGELOG.md index f298cfc0f..d30b292d7 100644 --- a/languages/golang/CHANGELOG.md +++ b/languages/golang/CHANGELOG.md @@ -19,12 +19,14 @@ listed under **Breaking**. Releases are tagged `languages/golang/vX.Y.Z`. `GetDocumentPtr`), so 64-bit integers and finite `float64`s round-trip exactly. A pointer to one takes `null=sql`. `stashgen` refuses, when it runs, a type `encoding/json` cannot carry both ways: an interface such as - `any`, an unexported struct field or one tagged `json:"-"`, a channel, a - function, a scalar, or a `[]byte`. The `Fields` entry's operands are Go + `any`, a struct field `encoding/json` drops (unexported, tagged + `json:"-"`, or two writing one member), a one-way `MarshalJSON` or + `MarshalText`, a channel, a function, a scalar, or a `[]byte`. The `Fields` entry's operands are Go values now: `Contains`, `ContainedBy`, `EqAt`, `Lt`, `Lte`, `Gt`, `Gte` and `Between` take `any`, a `json.RawMessage` crossing as the JSON text it is and anything else - marshalled (`gensupport.JSONText`); a plain `[]byte` is refused. A zero + marshalled (`gensupport.JSONText`); a `[]byte` or a type defined over + one is refused, and so is a Go string as a containment needle. A zero struct field without `omitempty` narrows a containment query, so `Contains` on a field whose type holds a struct refuses a needle of that type (`gensupport.NewDocumentField`); the generated doc comments say how diff --git a/languages/golang/cmd/stashgen/README.md b/languages/golang/cmd/stashgen/README.md index 8f1349338..4e753cf98 100644 --- a/languages/golang/cmd/stashgen/README.md +++ b/languages/golang/cmd/stashgen/README.md @@ -143,7 +143,8 @@ people, err := customers.Decrypt(ctx, cipher, rows) // each Profile is a Profile The generated code marshals a struct, map or slice type with `encoding/json` when it seals the field, and unmarshals the opened document into the declared type. It calls `encoding/json` directly, never through `map[string]any`, so an `int64`, a `uint64` and a finite `float64` come back exactly. -`stashgen` refuses a type that `encoding/json` cannot carry both ways: an interface such as `any` anywhere in it (it reads numbers back as `float64`), an unexported struct field or one tagged `json:"-"` (each is dropped), a channel or a function. +`stashgen` refuses a type that `encoding/json` cannot carry both ways: an interface such as `any` anywhere in it (it reads numbers back as `float64`); a struct field that `encoding/json` drops (unexported, tagged `json:"-"`, or one of two that write the same member); a `MarshalJSON` or `MarshalText` with no matching unmarshal method; a channel or a function. +An unexported embedded struct is fine: its fields are promoted and stored. It refuses a string, a number and a bool too: a document is a struct, map or slice. A `[]byte` is refused, since `encoding/json` writes it as base64: declare JSON text as `json.RawMessage`. The engine parses the JSON text, so an object that names a member twice and an integer outside the 64-bit range are refused with `encrypt.ErrEncoding`. @@ -172,7 +173,9 @@ An ordering compares the value `->` extracts at its path, and EQL's `<` compares `OpenOne` reads one value and refuses a read whose path has a wildcard before anything is opened; open a wildcard read with `Open`. A needle and a value are Go values: a `json.RawMessage` of JSON text crosses as it is, and anything else is marshalled with `encoding/json`. `EqAt` and the orderings take a scalar: a string, a number, a bool, or `nil` for JSON `null`. -A plain `[]byte` is refused rather than written as base64. +A `[]byte`, or a type defined over one, is refused rather than written as base64. +A Go string is refused as a `Contains` or `ContainedBy` needle, since it would be one JSON string: pass JSON text as `json.RawMessage`. +For `EqAt` and the orderings a Go string is the string value itself. A document contains a needle when it holds every member the needle writes, and `encoding/json` writes a struct field that has no `omitempty` even when it is zero. So a zero field narrows the query to the documents that hold that zero: `Profile{Name: "Alice"}` also asks for `"age": 0`. Give a needle struct pointer fields with `omitempty`, as `ProfileNeedle` above, or use a map. diff --git a/languages/golang/cmd/stashgen/main_test.go b/languages/golang/cmd/stashgen/main_test.go index 1d57f5064..79621103d 100644 --- a/languages/golang/cmd/stashgen/main_test.go +++ b/languages/golang/cmd/stashgen/main_test.go @@ -304,7 +304,7 @@ func TestRunGeneratesTypedJSONDocuments(t *testing.T) { t.Skip(err) } _ = checker.Close() - types := "type Profile struct {\n\tName string `json:\"name\"`\n\tAge int64 `json:\"age\"`\n\tVisits []Visit `json:\"visits\"`\n}\n\ntype Visit struct {\n\tAt int64 `json:\"at\"`\n}\n\ntype Prefs struct {\n\tTheme string `json:\"theme\"`\n}\n\ntype Hidden struct {\n\tName string\n\tsecret string\n}\n\ntype Loose struct {\n\tExtra any\n}\n\ntype Dropped struct {\n\tName string\n\tToken string `json:\"-\"`\n}\n" + types := "type Profile struct {\n\tName string `json:\"name\"`\n\tAge int64 `json:\"age\"`\n\tVisits []Visit `json:\"visits\"`\n}\n\ntype Visit struct {\n\tAt int64 `json:\"at\"`\n}\n\ntype Prefs struct {\n\tTheme string `json:\"theme\"`\n}\n\ntype Hidden struct {\n\tName string\n\tsecret string\n}\n\ntype Loose struct {\n\tExtra any\n}\n\ntype Dropped struct {\n\tName string\n\tToken string `json:\"-\"`\n}\n\ntype Twice struct {\n\tName string `json:\"name\"`\n\tAlias string `json:\"name\"`\n}\n\ntype Label struct{ V string }\n\nfunc (l Label) MarshalText() ([]byte, error) { return []byte(l.V), nil }\n\ntype base struct {\n\tKind string `json:\"kind\"`\n}\n\ntype Embeds struct {\n\tbase\n\tName string `json:\"name\"`\n}\n\ntype Stamped struct {\n\ttime.Time\n\tName string\n}\n" generate := func(fields string) (string, string, int) { source := "package users\n\nimport (\n\t\"encoding/json\"\n\t\"time\"\n)\n\nvar (\n\t_ json.RawMessage\n\t_ time.Time\n)\n\n" + types + "\ntype Customer struct {\n\t_ struct{} `stash:\"context=customers\"`\n\tID int64 `stash:\"id,passthrough\"`\n" + fields + "\n}\n" dir := writeModule(t, map[string]string{"model.go": source}) @@ -313,6 +313,10 @@ func TestRunGeneratesTypedJSONDocuments(t *testing.T) { written, _ := os.ReadFile(filepath.Join(dir, "customer_stash.go")) return string(written), stderr.String(), code } + // An unexported embedded struct's fields are promoted, and stored. + if _, stderr, code := generate("\tProfile Embeds `stash:\"profile,encrypt_into=JsonSearch\"`"); code != 0 { + t.Fatalf("an embedded unexported struct: exit %d\n%s", code, stderr) + } written, stderr, code := generate("\tProfile Profile `stash:\"profile,encrypt_into=JsonSearch(ordered=$.age)\"`\n\tPrefs *Prefs `stash:\"prefs,encrypt_into=JsonSearch,null=sql\"`\n\tLabels map[string]string `stash:\"labels,encrypt_into=JsonSearch\"`\n\tScores []int64 `stash:\"scores,encrypt_into=JsonSearch\"`") if code != 0 { t.Fatalf("exit %d\n%s", code, stderr) @@ -337,10 +341,11 @@ func TestRunGeneratesTypedJSONDocuments(t *testing.T) { `Scores: newScoresField(gensupport.NewField[[]int64](declaration, "scores")),`, "func (f ProfileField) Encrypt(ctx context.Context, c *encrypt.Cipher, v Profile) (eql.JSONSearch, error)", "func (f ProfileField) Contains(ctx context.Context, c *encrypt.Cipher, needle any) (eql.JSONSearchQuery, error)", - "A\n// needle of the field's own type, Profile, is refused for that reason.", + "A needle of the field's own type, Profile, is refused for that reason.", "func (p ProfileFieldPath) Gt(ctx context.Context, c *encrypt.Cipher, v any) (eql.JSONSearchLeafQuery, error)", } { - if !strings.Contains(written, want) { + // A doc comment is wrapped: read it as one line. + if !strings.Contains(written, want) && !strings.Contains(strings.ReplaceAll(written, "\n// ", " "), want) { t.Errorf("the generated file lacks %q", want) } } @@ -357,8 +362,11 @@ func TestRunGeneratesTypedJSONDocuments(t *testing.T) { {"an interface", "\tProfile map[string]any `stash:\"profile,encrypt_into=JsonSearch\"`", "holds an interface"}, {"an interface in a struct", "\tProfile Loose `stash:\"profile,encrypt_into=JsonSearch\"`", "holds an interface"}, {"an unexported field", "\tProfile Hidden `stash:\"profile,encrypt_into=JsonSearch\"`", "unexported field secret"}, - {"a field json drops", "\tProfile []Dropped `stash:\"profile,encrypt_into=JsonSearch\"`", "has a field Token tagged json:\"-\""}, - {"a channel", "\tProfile []chan int `stash:\"profile,encrypt_into=JsonSearch\"`", "is not a type JSON carries"}, + {"a field json drops", "\tProfile []Dropped `stash:\"profile,encrypt_into=JsonSearch\"`", "field Token tagged json:\"-\", which encoding/json drops"}, + {"two fields, one member", "\tProfile Twice `stash:\"profile,encrypt_into=JsonSearch\"`", "fields Name and Alias both write the JSON member \"name\""}, + {"a codec one way", "\tProfile map[string]Label `stash:\"profile,encrypt_into=JsonSearch\"`", "has MarshalText but no UnmarshalText"}, + {"a codec taken from an embedded field", "\tProfile Stamped `stash:\"profile,encrypt_into=JsonSearch\"`", "takes MarshalJSON from a field it embeds"}, + {"a channel", "\tProfile []chan int `stash:\"profile,encrypt_into=JsonSearch\"`", "holds a chan int, which JSON does not carry"}, {"a pointer to a pointer", "\tProfile **Profile `stash:\"profile,encrypt_into=JsonSearch,null=sql\"`", "a pointer to a pointer"}, {"a nil document that does not say null=sql", "\tPrefs *Prefs `stash:\"prefs,encrypt_into=JsonSearch\"`", "null=sql"}, {"a sealed null", "\tPrefs *Prefs `stash:\"prefs,encrypt_into=JsonSearch,null=sealed\"`", "Customer.Prefs"}, diff --git a/languages/golang/encrypt/decode_json_test.go b/languages/golang/encrypt/decode_json_test.go index 4dca067ae..935d6ddb5 100644 --- a/languages/golang/encrypt/decode_json_test.go +++ b/languages/golang/encrypt/decode_json_test.go @@ -36,6 +36,14 @@ func TestDecodeJSONReadsTheCallersType(t *testing.T) { if p, err := encrypt.DecodeJSON[*visit](json.RawMessage(`null`)); err != nil || p != nil { t.Fatalf("null: %v %v", p, err) } + // Trailing text that is not JSON is told apart from a second value, and + // an interface type is named. + if _, err := encrypt.DecodeJSON[map[string]int](json.RawMessage(`{"a":1}}`)); err == nil || strings.Contains(err.Error(), "more than one") { + t.Fatalf("trailing junk: %v", err) + } + if _, err := encrypt.DecodeJSON[any](json.RawMessage(``)); err == nil || !strings.Contains(err.Error(), "interface {}") { + t.Fatalf("an interface type is not named: %v", err) + } for name, text := range map[string]string{ "an unfound value": ``, "not JSON": `{"secret-value"`, diff --git a/languages/golang/encrypt/doc.go b/languages/golang/encrypt/doc.go index e03a70762..e861ec229 100644 --- a/languages/golang/encrypt/doc.go +++ b/languages/golang/encrypt/doc.go @@ -216,6 +216,9 @@ // q, err := customers.Fields.Profile.Contains(ctx, cipher, map[string]any{"tier": "gold"}) // older, err := customers.Fields.Profile.At.Age.Gt(ctx, cipher, 40) // +// A Go string is the string value itself, never parsed. So Contains and +// ContainedBy refuse a string needle; put JSON text in a json.RawMessage. +// // A document contains a needle when it holds every member the needle writes. // encoding/json writes a struct field that has no omitempty even when the // field is zero. So a zero field narrows the query to documents that hold diff --git a/languages/golang/encrypt/gensupport/json.go b/languages/golang/encrypt/gensupport/json.go index ac97a96f7..df23ac51d 100644 --- a/languages/golang/encrypt/gensupport/json.go +++ b/languages/golang/encrypt/gensupport/json.go @@ -2,9 +2,11 @@ package gensupport import ( "context" + "encoding" "encoding/json" "errors" "fmt" + "reflect" "github.com/cipherstash/stack/languages/golang/encrypt" "github.com/cipherstash/stack/languages/golang/internal/record" @@ -30,11 +32,13 @@ import ( // encoding/json's own text quotes a NaN, and a MarshalJSON's error may hold // anything. func JSONText(v any) ([]byte, error) { - switch v := v.(type) { - case json.RawMessage: - return v, nil - case []byte: - return nil, fmt.Errorf("%w: a []byte is not JSON text to encoding/json, which writes it as a base64 string; convert JSON text to json.RawMessage", encrypt.ErrEncoding) + if raw, ok := v.(json.RawMessage); ok { + return raw, nil + } + if writesItself(v) == "" && kindOf(v) == reflect.Slice && reflect.TypeOf(v).Elem().Kind() == reflect.Uint8 { + // A []byte, or a type defined over one (type Doc []byte, or + // json.RawMessage redefined, which drops its MarshalJSON). + return nil, fmt.Errorf("%w: a %T is not JSON text to encoding/json, which writes it as a base64 string; convert JSON text to json.RawMessage", encrypt.ErrEncoding, v) } text, err := json.Marshal(v) if err == nil { @@ -54,6 +58,29 @@ func JSONText(v any) ([]byte, error) { return nil, fmt.Errorf("%w: a %T does not write as JSON", encrypt.ErrEncoding, v) } +// writesItself names the encoding/json interface v implements to write +// itself, or "" for none. +func writesItself(v any) string { + switch v.(type) { + case json.Marshaler: + return "json.Marshaler" + case encoding.TextMarshaler: + return "encoding.TextMarshaler" + } + return "" +} + +// kindOf is the kind of v's dynamic type, or reflect.Invalid for nil: the +// one place the JSON operands look at a type by kind, since a type switch +// cannot name every type defined over []byte or string. +func kindOf(v any) reflect.Kind { + t := reflect.TypeOf(v) + if t == nil { + return reflect.Invalid + } + return t.Kind() +} + // GetDocument reads one opened document field into the Go type T the struct // declares, with [encrypt.DecodeJSON]: the explicit encoding/json read that // mirrors the write [JSONText] made. A json.RawMessage field is read with @@ -129,10 +156,17 @@ func (f Field[T]) jsonOperand(v any) ([]byte, error) { // JSONQuery derives a containment operator's query on a JSON field // (contains, contained_by): the needle's projection, Q. The needle is any -// value [JSONText] writes. Contains on a field of [NewDocumentField] +// value [JSONText] writes, other than a string, which is refused: one JSON +// string is never what a containment needle means, and JSON text in a Go +// string is the likely mistake. Contains on a field of [NewDocumentField] // refuses a needle of the field's own type, T or *T: its zero-valued // fields would narrow the query. func JSONQuery[Q ~[]byte, T any](ctx context.Context, f Field[T], c *encrypt.Cipher, operator string, needle any) (Q, error) { + // A Go string is one JSON string, and a document of members never + // contains one: it is almost always JSON text meant as the needle. + if kindOf(needle) == reflect.String && writesItself(needle) == "" { + return nil, fmt.Errorf("%w: gensupport: %s: a %T needle is one JSON string, not JSON text; pass JSON text as json.RawMessage, or a map or struct", encrypt.ErrEncoding, f.name, needle) + } if f.holdsStruct && operator == "contains" { switch needle.(type) { case T, *T: diff --git a/languages/golang/encrypt/internal/testusers/customer_stash.go b/languages/golang/encrypt/internal/testusers/customer_stash.go index 3da2a6860..d2d8347c5 100644 --- a/languages/golang/encrypt/internal/testusers/customer_stash.go +++ b/languages/golang/encrypt/internal/testusers/customer_stash.go @@ -140,28 +140,30 @@ func (f CustomerProfileField) Encrypt(ctx context.Context, c *encrypt.Cipher, v } // Contains derives the query for documents that contain needle: col @> query. -// needle is a json.RawMessage of JSON text, or any value encoding/json writes, -// such as a map or a struct. Each member it writes must be in the document, so -// a zero-valued struct field narrows the query to documents holding that zero: -// give a needle struct pointer fields, or omitempty, or use a map. Each array -// element of the needle must sit inside one element of the document's array. +// needle is a json.RawMessage of JSON text, or a map, struct or slice that +// encoding/json writes; a Go string is refused, as one JSON string is not JSON +// text. Each member it writes must be in the document, so a zero-valued struct +// field narrows the query to documents holding that zero: give a needle struct +// pointer fields, or omitempty, or use a map. Each array element of the needle +// must sit inside one element of the document's array. func (f CustomerProfileField) Contains(ctx context.Context, c *encrypt.Cipher, needle any) (eql.JSONSearchQuery, error) { return gensupport.JSONQuery[eql.JSONSearchQuery](ctx, f.field, c, "contains", needle) } // ContainedBy derives the query for documents contained by needle: col <@ -// query. needle is a json.RawMessage of JSON text, or any value encoding/json -// writes; here a zero-valued field widens the query, admitting documents that -// hold that zero. +// query. needle is a json.RawMessage of JSON text, or a map, struct or slice +// that encoding/json writes (not a Go string); here a zero-valued field widens +// the query, admitting documents that hold that zero. func (f CustomerProfileField) ContainedBy(ctx context.Context, c *encrypt.Cipher, needle any) (eql.JSONSearchQuery, error) { return gensupport.JSONQuery[eql.JSONSearchQuery](ctx, f.field, c, "contained_by", needle) } // EqAt derives the query for documents holding the scalar v at path: col @> // query. v is a Go string, number or bool, nil for JSON null, or a -// json.RawMessage of JSON text. A wildcard step matches when any element holds -// it; a position ([2]) is refused, since the stored selector reads every -// position as the wildcard. +// json.RawMessage of JSON text; a Go string is the JSON string it holds, never +// parsed as JSON. A wildcard step matches when any element holds it; a position +// ([2]) is refused, since the stored selector reads every position as the +// wildcard. func (f CustomerProfileField) EqAt(ctx context.Context, c *encrypt.Cipher, path string, v any) (eql.JSONSearchQuery, error) { return gensupport.JSONQueryAt[eql.JSONSearchQuery](ctx, f.field, c, "eq_at", path, v) } @@ -251,28 +253,30 @@ func (f CustomerPrefsField) Encrypt(ctx context.Context, c *encrypt.Cipher, v js } // Contains derives the query for documents that contain needle: col @> query. -// needle is a json.RawMessage of JSON text, or any value encoding/json writes, -// such as a map or a struct. Each member it writes must be in the document, so -// a zero-valued struct field narrows the query to documents holding that zero: -// give a needle struct pointer fields, or omitempty, or use a map. Each array -// element of the needle must sit inside one element of the document's array. +// needle is a json.RawMessage of JSON text, or a map, struct or slice that +// encoding/json writes; a Go string is refused, as one JSON string is not JSON +// text. Each member it writes must be in the document, so a zero-valued struct +// field narrows the query to documents holding that zero: give a needle struct +// pointer fields, or omitempty, or use a map. Each array element of the needle +// must sit inside one element of the document's array. func (f CustomerPrefsField) Contains(ctx context.Context, c *encrypt.Cipher, needle any) (eql.JSONSearchQuery, error) { return gensupport.JSONQuery[eql.JSONSearchQuery](ctx, f.field, c, "contains", needle) } // ContainedBy derives the query for documents contained by needle: col <@ -// query. needle is a json.RawMessage of JSON text, or any value encoding/json -// writes; here a zero-valued field widens the query, admitting documents that -// hold that zero. +// query. needle is a json.RawMessage of JSON text, or a map, struct or slice +// that encoding/json writes (not a Go string); here a zero-valued field widens +// the query, admitting documents that hold that zero. func (f CustomerPrefsField) ContainedBy(ctx context.Context, c *encrypt.Cipher, needle any) (eql.JSONSearchQuery, error) { return gensupport.JSONQuery[eql.JSONSearchQuery](ctx, f.field, c, "contained_by", needle) } // EqAt derives the query for documents holding the scalar v at path: col @> // query. v is a Go string, number or bool, nil for JSON null, or a -// json.RawMessage of JSON text. A wildcard step matches when any element holds -// it; a position ([2]) is refused, since the stored selector reads every -// position as the wildcard. +// json.RawMessage of JSON text; a Go string is the JSON string it holds, never +// parsed as JSON. A wildcard step matches when any element holds it; a position +// ([2]) is refused, since the stored selector reads every position as the +// wildcard. func (f CustomerPrefsField) EqAt(ctx context.Context, c *encrypt.Cipher, path string, v any) (eql.JSONSearchQuery, error) { return gensupport.JSONQueryAt[eql.JSONSearchQuery](ctx, f.field, c, "eq_at", path, v) } diff --git a/languages/golang/encrypt/internal/testusers/subscriber_stash.go b/languages/golang/encrypt/internal/testusers/subscriber_stash.go index fe86b2989..3e7f7a3f0 100644 --- a/languages/golang/encrypt/internal/testusers/subscriber_stash.go +++ b/languages/golang/encrypt/internal/testusers/subscriber_stash.go @@ -150,30 +150,31 @@ func (f SubscriberProfileField) Encrypt(ctx context.Context, c *encrypt.Cipher, } // Contains derives the query for documents that contain needle: col @> query. -// needle is a json.RawMessage of JSON text, or any value encoding/json writes, -// such as a map or a struct. Each member it writes must be in the document, so -// a zero-valued struct field narrows the query to documents holding that zero: -// give a needle struct pointer fields, or omitempty, or use a map. Each array -// element of the needle must sit inside one element of the document's array. A -// needle of the field's own type, SubscriberProfile, is refused for that -// reason. +// needle is a json.RawMessage of JSON text, or a map, struct or slice that +// encoding/json writes; a Go string is refused, as one JSON string is not JSON +// text. Each member it writes must be in the document, so a zero-valued struct +// field narrows the query to documents holding that zero: give a needle struct +// pointer fields, or omitempty, or use a map. Each array element of the needle +// must sit inside one element of the document's array. A needle of the field's +// own type, SubscriberProfile, is refused for that reason. func (f SubscriberProfileField) Contains(ctx context.Context, c *encrypt.Cipher, needle any) (eql.JSONSearchQuery, error) { return gensupport.JSONQuery[eql.JSONSearchQuery](ctx, f.field, c, "contains", needle) } // ContainedBy derives the query for documents contained by needle: col <@ -// query. needle is a json.RawMessage of JSON text, or any value encoding/json -// writes; here a zero-valued field widens the query, admitting documents that -// hold that zero. +// query. needle is a json.RawMessage of JSON text, or a map, struct or slice +// that encoding/json writes (not a Go string); here a zero-valued field widens +// the query, admitting documents that hold that zero. func (f SubscriberProfileField) ContainedBy(ctx context.Context, c *encrypt.Cipher, needle any) (eql.JSONSearchQuery, error) { return gensupport.JSONQuery[eql.JSONSearchQuery](ctx, f.field, c, "contained_by", needle) } // EqAt derives the query for documents holding the scalar v at path: col @> // query. v is a Go string, number or bool, nil for JSON null, or a -// json.RawMessage of JSON text. A wildcard step matches when any element holds -// it; a position ([2]) is refused, since the stored selector reads every -// position as the wildcard. +// json.RawMessage of JSON text; a Go string is the JSON string it holds, never +// parsed as JSON. A wildcard step matches when any element holds it; a position +// ([2]) is refused, since the stored selector reads every position as the +// wildcard. func (f SubscriberProfileField) EqAt(ctx context.Context, c *encrypt.Cipher, path string, v any) (eql.JSONSearchQuery, error) { return gensupport.JSONQueryAt[eql.JSONSearchQuery](ctx, f.field, c, "eq_at", path, v) } @@ -263,29 +264,31 @@ func (f SubscriberPrefsField) Encrypt(ctx context.Context, c *encrypt.Cipher, v } // Contains derives the query for documents that contain needle: col @> query. -// needle is a json.RawMessage of JSON text, or any value encoding/json writes, -// such as a map or a struct. Each member it writes must be in the document, so -// a zero-valued struct field narrows the query to documents holding that zero: -// give a needle struct pointer fields, or omitempty, or use a map. Each array -// element of the needle must sit inside one element of the document's array. A -// needle of the field's own type, SubscriberPrefs, is refused for that reason. +// needle is a json.RawMessage of JSON text, or a map, struct or slice that +// encoding/json writes; a Go string is refused, as one JSON string is not JSON +// text. Each member it writes must be in the document, so a zero-valued struct +// field narrows the query to documents holding that zero: give a needle struct +// pointer fields, or omitempty, or use a map. Each array element of the needle +// must sit inside one element of the document's array. A needle of the field's +// own type, SubscriberPrefs, is refused for that reason. func (f SubscriberPrefsField) Contains(ctx context.Context, c *encrypt.Cipher, needle any) (eql.JSONSearchQuery, error) { return gensupport.JSONQuery[eql.JSONSearchQuery](ctx, f.field, c, "contains", needle) } // ContainedBy derives the query for documents contained by needle: col <@ -// query. needle is a json.RawMessage of JSON text, or any value encoding/json -// writes; here a zero-valued field widens the query, admitting documents that -// hold that zero. +// query. needle is a json.RawMessage of JSON text, or a map, struct or slice +// that encoding/json writes (not a Go string); here a zero-valued field widens +// the query, admitting documents that hold that zero. func (f SubscriberPrefsField) ContainedBy(ctx context.Context, c *encrypt.Cipher, needle any) (eql.JSONSearchQuery, error) { return gensupport.JSONQuery[eql.JSONSearchQuery](ctx, f.field, c, "contained_by", needle) } // EqAt derives the query for documents holding the scalar v at path: col @> // query. v is a Go string, number or bool, nil for JSON null, or a -// json.RawMessage of JSON text. A wildcard step matches when any element holds -// it; a position ([2]) is refused, since the stored selector reads every -// position as the wildcard. +// json.RawMessage of JSON text; a Go string is the JSON string it holds, never +// parsed as JSON. A wildcard step matches when any element holds it; a position +// ([2]) is refused, since the stored selector reads every position as the +// wildcard. func (f SubscriberPrefsField) EqAt(ctx context.Context, c *encrypt.Cipher, path string, v any) (eql.JSONSearchQuery, error) { return gensupport.JSONQueryAt[eql.JSONSearchQuery](ctx, f.field, c, "eq_at", path, v) } @@ -333,28 +336,30 @@ func (f SubscriberLabelsField) Encrypt(ctx context.Context, c *encrypt.Cipher, v } // Contains derives the query for documents that contain needle: col @> query. -// needle is a json.RawMessage of JSON text, or any value encoding/json writes, -// such as a map or a struct. Each member it writes must be in the document, so -// a zero-valued struct field narrows the query to documents holding that zero: -// give a needle struct pointer fields, or omitempty, or use a map. Each array -// element of the needle must sit inside one element of the document's array. +// needle is a json.RawMessage of JSON text, or a map, struct or slice that +// encoding/json writes; a Go string is refused, as one JSON string is not JSON +// text. Each member it writes must be in the document, so a zero-valued struct +// field narrows the query to documents holding that zero: give a needle struct +// pointer fields, or omitempty, or use a map. Each array element of the needle +// must sit inside one element of the document's array. func (f SubscriberLabelsField) Contains(ctx context.Context, c *encrypt.Cipher, needle any) (eql.JSONSearchQuery, error) { return gensupport.JSONQuery[eql.JSONSearchQuery](ctx, f.field, c, "contains", needle) } // ContainedBy derives the query for documents contained by needle: col <@ -// query. needle is a json.RawMessage of JSON text, or any value encoding/json -// writes; here a zero-valued field widens the query, admitting documents that -// hold that zero. +// query. needle is a json.RawMessage of JSON text, or a map, struct or slice +// that encoding/json writes (not a Go string); here a zero-valued field widens +// the query, admitting documents that hold that zero. func (f SubscriberLabelsField) ContainedBy(ctx context.Context, c *encrypt.Cipher, needle any) (eql.JSONSearchQuery, error) { return gensupport.JSONQuery[eql.JSONSearchQuery](ctx, f.field, c, "contained_by", needle) } // EqAt derives the query for documents holding the scalar v at path: col @> // query. v is a Go string, number or bool, nil for JSON null, or a -// json.RawMessage of JSON text. A wildcard step matches when any element holds -// it; a position ([2]) is refused, since the stored selector reads every -// position as the wildcard. +// json.RawMessage of JSON text; a Go string is the JSON string it holds, never +// parsed as JSON. A wildcard step matches when any element holds it; a position +// ([2]) is refused, since the stored selector reads every position as the +// wildcard. func (f SubscriberLabelsField) EqAt(ctx context.Context, c *encrypt.Cipher, path string, v any) (eql.JSONSearchQuery, error) { return gensupport.JSONQueryAt[eql.JSONSearchQuery](ctx, f.field, c, "eq_at", path, v) } diff --git a/languages/golang/encrypt/json.go b/languages/golang/encrypt/json.go index 2c06f4ea2..a79384ef6 100644 --- a/languages/golang/encrypt/json.go +++ b/languages/golang/encrypt/json.go @@ -344,37 +344,48 @@ func DecodeJSON[T any](text json.RawMessage) (T, error) { dec.UseNumber() if err := dec.Decode(&out); err != nil { var zero T - return zero, decodeError(err, out) + return zero, decodeError(err, typeName[T]()) } // One value, and nothing after it: Decode reads the first of a stream. - if _, err := dec.Token(); !errors.Is(err, io.EOF) { + switch _, err := dec.Token(); { + case errors.Is(err, io.EOF): + return out, nil + case err == nil: var zero T - return zero, fmt.Errorf("%w: the JSON text holds more than one value, and reads into one %T", ErrEncoding, zero) + return zero, fmt.Errorf("%w: the JSON text holds more than one value, and reads into one %s", ErrEncoding, typeName[T]()) + default: + var zero T + return zero, fmt.Errorf("%w: the text after its first JSON value is not JSON, and reads into no %s", ErrEncoding, typeName[T]()) } - return out, nil +} + +// typeName is T as Go source, an interface type included: %T of a T +// holding nil prints , and a *T names its element. +func typeName[T any]() string { + return strings.TrimPrefix(fmt.Sprintf("%T", (*T)(nil)), "*") } // decodeError is encoding/json's decode error without the text it quotes: // a SyntaxError names the character it met and an UnmarshalTypeError the // number it could not store, both plaintext. It keeps the kind of JSON value // and the Go type that did not fit. -func decodeError(err error, out any) error { +func decodeError(err error, out string) error { var typeErr *json.UnmarshalTypeError switch { case errors.As(err, &typeErr): // Value is "number", "number 1.5", "string", "bool", "array" or // "object": its first word is the kind, the rest the value. kind, _, _ := strings.Cut(typeErr.Value, " ") - return fmt.Errorf("%w: a JSON %s does not read into the Go type %s, in a %T", ErrEncoding, kind, typeErr.Type, out) + return fmt.Errorf("%w: a JSON %s does not read into the Go type %s, in a %s", ErrEncoding, kind, typeErr.Type, out) case errors.Is(err, io.EOF), errors.Is(err, io.ErrUnexpectedEOF): - return fmt.Errorf("%w: no JSON value to read into a %T (an unfound JSONValue holds none: check Found)", ErrEncoding, out) + return fmt.Errorf("%w: no JSON value to read into a %s (an unfound JSONValue holds none: check Found)", ErrEncoding, out) } var syntax *json.SyntaxError if errors.As(err, &syntax) { - return fmt.Errorf("%w: the text is not JSON (at byte %d), and reads into no %T", ErrEncoding, syntax.Offset, out) + return fmt.Errorf("%w: the text is not JSON (at byte %d), and reads into no %s", ErrEncoding, syntax.Offset, out) } // An UnmarshalJSON or UnmarshalText of the caller's type failed, or // encoding/json refused the target (a non-pointer, say): its text may // hold the value, so only the type is named. - return fmt.Errorf("%w: the JSON text does not read into a %T", ErrEncoding, out) + return fmt.Errorf("%w: the JSON text does not read into a %s", ErrEncoding, out) } diff --git a/languages/golang/encrypt/json_typed_test.go b/languages/golang/encrypt/json_typed_test.go index 8249053d2..3e5647f41 100644 --- a/languages/golang/encrypt/json_typed_test.go +++ b/languages/golang/encrypt/json_typed_test.go @@ -165,10 +165,24 @@ func TestASubscribersTypedNeedlesDeriveTheRawQuery(t *testing.T) { } // What encoding/json cannot write, and a []byte, are refused, naming // no value. + type doc []byte + type redefined json.RawMessage + type text string for name, err := range map[string]error{ "[]byte": func() error { _, err := f.Contains(ctx, cipher, []byte(`{"tier":"secret"}`)); return err }(), - "NaN": func() error { _, err := f.At.Age.Lt(ctx, cipher, math.NaN()); return err }(), - "chan": func() error { _, err := f.EqAt(ctx, cipher, "$.name", make(chan int)); return err }(), + // A type defined over []byte, json.RawMessage included, has no + // MarshalJSON and would be written as base64. + "defined []byte": func() error { _, err := f.EqAt(ctx, cipher, "$.name", doc(`"secret"`)); return err }(), + "redefined RawMessage": func() error { + _, err := f.Contains(ctx, cipher, redefined(`{"tier":"secret"}`)) + return err + }(), + // JSON text in a Go string would be one JSON string: refused as a + // needle, which is never what it means. + "string needle": func() error { _, err := f.Contains(ctx, cipher, `{"tier":"secret"}`); return err }(), + "defined string needle": func() error { _, err := f.ContainedBy(ctx, cipher, text(`{"tier":"secret"}`)); return err }(), + "NaN": func() error { _, err := f.At.Age.Lt(ctx, cipher, math.NaN()); return err }(), + "chan": func() error { _, err := f.EqAt(ctx, cipher, "$.name", make(chan int)); return err }(), } { if !errors.Is(err, encrypt.ErrEncoding) || strings.Contains(err.Error(), "secret") { t.Errorf("%s: %v", name, err) diff --git a/languages/golang/stashgen/emit.go b/languages/golang/stashgen/emit.go index 0f9761858..c25fe9b29 100644 --- a/languages/golang/stashgen/emit.go +++ b/languages/golang/stashgen/emit.go @@ -1123,7 +1123,7 @@ func (w *writer) jsonEntry(g genField) { recv := "func (f " + g.fieldType + ") " if answered["contains"] { w.nl() - doc := "Contains derives the query for documents that contain needle: col @> query. needle is a json.RawMessage of JSON text, or any value encoding/json writes, such as a map or a struct. Each member it writes must be in the document, so a zero-valued struct field narrows the query to documents holding that zero: give a needle struct pointer fields, or omitempty, or use a map. Each array element of the needle must sit inside one element of the document's array." + doc := "Contains derives the query for documents that contain needle: col @> query. needle is a json.RawMessage of JSON text, or a map, struct or slice that encoding/json writes; a Go string is refused, as one JSON string is not JSON text. Each member it writes must be in the document, so a zero-valued struct field narrows the query to documents holding that zero: give a needle struct pointer fields, or omitempty, or use a map. Each array element of the needle must sit inside one element of the document's array." if g.holdsStruct { doc += " A needle of the field's own type, " + g.valueExpr + ", is refused for that reason." } @@ -1134,14 +1134,14 @@ func (w *writer) jsonEntry(g genField) { } if answered["contained_by"] { w.nl() - w.b.WriteString(wrapComment("ContainedBy derives the query for documents contained by needle: col <@ query. needle is a json.RawMessage of JSON text, or any value encoding/json writes; here a zero-valued field widens the query, admitting documents that hold that zero.", 80)) + w.b.WriteString(wrapComment("ContainedBy derives the query for documents contained by needle: col <@ query. needle is a json.RawMessage of JSON text, or a map, struct or slice that encoding/json writes (not a Go string); here a zero-valued field widens the query, admitting documents that hold that zero.", 80)) w.p("%sContainedBy(ctx context.Context, c *encrypt.Cipher, needle any) (%s, error) {", recv, needle) w.p("\treturn gensupport.JSONQuery[%s](ctx, f.field, c, \"contained_by\", needle)", needle) w.p("}") } if answered["eq_at"] { w.nl() - w.b.WriteString(wrapComment("EqAt derives the query for documents holding the scalar v at path: col @> query. v is a Go string, number or bool, nil for JSON null, or a json.RawMessage of JSON text. A wildcard step matches when any element holds it; a position ([2]) is refused, since the stored selector reads every position as the wildcard.", 80)) + w.b.WriteString(wrapComment("EqAt derives the query for documents holding the scalar v at path: col @> query. v is a Go string, number or bool, nil for JSON null, or a json.RawMessage of JSON text; a Go string is the JSON string it holds, never parsed as JSON. A wildcard step matches when any element holds it; a position ([2]) is refused, since the stored selector reads every position as the wildcard.", 80)) w.p("%sEqAt(ctx context.Context, c *encrypt.Cipher, path string, v any) (%s, error) {", recv, needle) w.p("\treturn gensupport.JSONQueryAt[%s](ctx, f.field, c, \"eq_at\", path, v)", needle) w.p("}") diff --git a/languages/golang/stashgen/read.go b/languages/golang/stashgen/read.go index c745a66d2..d85b10d75 100644 --- a/languages/golang/stashgen/read.go +++ b/languages/golang/stashgen/read.go @@ -724,16 +724,16 @@ func (r *reader) buildFields(c *collected) error { // What encoding/json cannot carry both ways is refused here, when // the code is generated. holdsStruct := false - if field.Verb == VerbEncryptInto && r.storesDocument(field.EQLType) { + if eqlType := r.eqlType(field.EQLType); field.Verb == VerbEncryptInto && eqlType != nil && eqlType.wire == record.JSON { doc, docType := &field.GoType, cf.typ if doc.Kind == KindPointer && doc.Elem != nil { doc, docType = doc.Elem, docType.Underlying().(*types.Pointer).Elem() } if doc.Kind != KindJSON { - if reason := notDocument(docType); reason != "" { + var reason string + if reason, holdsStruct = notDocument(docType); reason != "" { return fieldErr(typeName, cf.goName, "%s seals a json document, and %s %s; %s", field.EQLType, r.typeExpr(docType), reason, documentTypes) } - holdsStruct = reaches(docType, func(u types.Type) bool { _, ok := u.(*types.Struct); return ok }, map[types.Type]bool{}) *doc = GoType{Name: doc.Name, Kind: KindJSONValue} } } @@ -786,12 +786,7 @@ func (r *reader) buildFields(c *collected) error { g.outputType = g.typeExpr g.outputs = []output{{name: "Value", typeExpr: g.typeExpr, pathType: g.pathType}} case VerbEncryptInto: - var eqlType *EQLType - for i := range r.eql { - if r.eql[i].Name == field.EQLType { - eqlType = &r.eql[i] - } - } + eqlType := r.eqlType(field.EQLType) if eqlType == nil { if len(r.eql) == 0 { return fieldErr(typeName, cf.goName, "EQL types are not available yet; this build of the engine produces none") @@ -864,10 +859,8 @@ func (r *reader) nullMode(field Field, tagged Null) (Null, error) { indexes = append(indexes, idx.Name) } if field.Verb == VerbEncryptInto { - for _, eqlType := range r.eql { - if eqlType.Name == field.EQLType { - indexes = eqlType.Indexes - } + if eqlType := r.eqlType(field.EQLType); eqlType != nil { + indexes = eqlType.Indexes } } indexed := false @@ -1165,115 +1158,184 @@ func notJSONEncodable(t types.Type, seen map[types.Type]bool) string { return "is not a type JSON carries" } -// storesDocument reports whether the engine's EQL type of that name stores -// a JSON document. -func (r *reader) storesDocument(name string) bool { - for _, t := range r.eql { - if t.Name == name { - return t.wire == record.JSON +// eqlType is the engine's EQL type of that name, or nil. +func (r *reader) eqlType(name string) *EQLType { + for i := range r.eql { + if r.eql[i].Name == name { + return &r.eql[i] } } - return false + return nil } // notDocument says why a Go type other than json.RawMessage is not a JSON // document that encoding/json writes and reads back to the same value, or -// "" when it is: a struct, map, slice or array whose every part -// encoding/json carries ([notJSONEncodable]) and holds no interface. -func notDocument(t types.Type) string { +// "" when it is; and whether the type holds a struct, whose zero fields +// would narrow a containment needle of that type. A document is a struct, +// map, slice or array, walked by [documentWalk]. +func notDocument(t types.Type) (string, bool) { if _, ok := valueKind(t); ok { - return "is one value, not a document" + return "is one value, not a document", false } switch u := t.Underlying().(type) { case *types.Basic: - return "is one JSON scalar, not a document (a string is a JSON string to encoding/json, not the JSON text it may hold)" + return "is one JSON scalar, not a document (a string is a JSON string to encoding/json, not the JSON text it may hold)", false case *types.Slice: if b, ok := u.Elem().Underlying().(*types.Basic); ok && b.Kind() == types.Byte { - return "is written by encoding/json as a base64 string, not as JSON text" + return "is written by encoding/json as a base64 string, not as JSON text", false } case *types.Pointer: - return "is a pointer to a pointer" + return "is a pointer to a pointer", false case *types.Struct, *types.Map, *types.Array: default: - return "is not a type JSON carries" - } - if reaches(t, func(u types.Type) bool { _, ok := u.(*types.Interface); return ok }, map[types.Type]bool{}) { - return "holds an interface, which encoding/json reads back as a map or a float64, not the value written" - } - if name := droppedField(t, map[types.Type]bool{}); name != "" { - return "has a field " + name + ` tagged json:"-", which encoding/json drops, so the stored document would not hold it` + return "is not a type JSON carries", false } - return notJSONEncodable(t, map[types.Type]bool{}) + w := documentWalk{seen: map[types.Type]bool{}} + return w.problem(t), w.holdsStruct } -// droppedField names the first exported struct field that t, or a type it -// reaches, tags json:"-", or "" for none: encoding/json writes no such -// field, so a document of the type would come back without it. A type that -// writes and reads itself is not looked into. -func droppedField(t types.Type, seen map[types.Type]bool) string { - if seen[t] { +// documentWalk checks a document type as encoding/json writes and reads +// it, once per type: what it writes must read back to the same value, so +// nothing is dropped and nothing changes type on the way. It refuses an +// interface (read back as a map or a float64), a value JSON does not carry, +// a map key encoding/json cannot write, and a struct field encoding/json +// drops: an unexported one, one tagged json:"-", and two that write one +// member. A type with both halves of a codec (MarshalJSON and +// UnmarshalJSON, or MarshalText and UnmarshalText) writes and reads itself +// and is not looked into; one half alone is refused. +type documentWalk struct { + seen map[types.Type]bool + holdsStruct bool +} + +// problem is why t is not carried both ways, as a phrase that follows the +// type's name ("holds an interface, ..."), or "". +func (w *documentWalk) problem(t types.Type) string { + if w.seen[t] { return "" } - seen[t] = true - if hasMethod(types.NewPointer(t), "UnmarshalJSON") && hasMethod(t, "MarshalJSON") { - return "" + w.seen[t] = true + if own, reason := ownCodec(t); reason != "" || own { + return reason } switch u := t.Underlying().(type) { + case *types.Basic: + if u.Info()&(types.IsBoolean|types.IsInteger|types.IsFloat|types.IsString) != 0 && u.Info()&types.IsUntyped == 0 { + return "" + } + return "holds a " + u.Name() + ", which JSON does not carry" + case *types.Interface: + return "holds an interface, which encoding/json reads back as a map or a float64, not the value written" case *types.Slice: - return droppedField(u.Elem(), seen) + return w.problem(u.Elem()) case *types.Array: - return droppedField(u.Elem(), seen) + return w.problem(u.Elem()) case *types.Pointer: - return droppedField(u.Elem(), seen) + return w.problem(u.Elem()) case *types.Map: - return droppedField(u.Elem(), seen) + if k, ok := u.Key().Underlying().(*types.Basic); !ok || k.Info()&(types.IsString|types.IsInteger) == 0 { + if text, _ := textForm(u.Key()); !text { + return "holds a map whose key is not a string, an integer or a type with MarshalText and UnmarshalText" + } + } + return w.problem(u.Elem()) case *types.Struct: - for i := range u.NumFields() { - fld := u.Field(i) - if fld.Exported() && reflect.StructTag(u.Tag(i)).Get("json") == "-" { - return fld.Name() + w.holdsStruct = true + return w.fields(u, map[string]string{}, map[types.Type]bool{}) + } + return "holds a " + types.TypeString(t.Underlying(), nil) + ", which JSON does not carry" +} + +// fields checks a struct's fields, and those an untagged embedded struct +// promotes into it, against names: the JSON member each field writes, so +// two that write one member are refused (encoding/json keeps one of them, +// or neither, silently). +func (w *documentWalk) fields(u *types.Struct, names map[string]string, embedding map[types.Type]bool) string { + for i := range u.NumFields() { + fld := u.Field(i) + tag := reflect.StructTag(u.Tag(i)).Get("json") + name, _, _ := strings.Cut(tag, ",") + if tag == "-" { + return "holds a struct with a field " + fld.Name() + ` tagged json:"-", which encoding/json drops` + } + if fld.Embedded() && name == "" { + et := fld.Type() + if p, ok := et.Underlying().(*types.Pointer); ok { + et = p.Elem() } - if name := droppedField(fld.Type(), seen); name != "" { - return name + if st, ok := et.Underlying().(*types.Struct); ok { + if own, _ := ownCodec(et); !own { + // encoding/json promotes an untagged embedded + // struct's fields, exported or not the struct is. + if embedding[et] { + continue + } + embedding[et] = true + if reason := w.fields(st, names, embedding); reason != "" { + return reason + } + continue + } } } + if !fld.Exported() { + return "holds a struct with an unexported field " + fld.Name() + ", which encoding/json drops" + } + if name == "" { + name = fld.Name() + } + if prior, dup := names[name]; dup { + return fmt.Sprintf("holds a struct whose fields %s and %s both write the JSON member %q, which encoding/json drops", prior, fld.Name(), name) + } + names[name] = fld.Name() + if reason := w.problem(fld.Type()); reason != "" { + return reason + } } return "" } -// reaches reports whether t, or a type its fields, elements or pointers -// reach, has an underlying type that match reports: a document type that -// holds a struct, say. A type that writes and reads itself (MarshalJSON and -// UnmarshalJSON) is not looked into, as encoding/json does not. -func reaches(t types.Type, match func(types.Type) bool, seen map[types.Type]bool) bool { - if seen[t] { - return false - } - seen[t] = true - if hasMethod(types.NewPointer(t), "UnmarshalJSON") && hasMethod(t, "MarshalJSON") { - return match(t.Underlying()) - } - u := t.Underlying() - if match(u) { - return true - } - switch u := u.(type) { - case *types.Slice: - return reaches(u.Elem(), match, seen) - case *types.Array: - return reaches(u.Elem(), match, seen) - case *types.Pointer: - return reaches(u.Elem(), match, seen) - case *types.Map: - return reaches(u.Elem(), match, seen) - case *types.Struct: - for i := range u.NumFields() { - if reaches(u.Field(i).Type(), match, seen) { - return true +// ownCodec reports whether t writes and reads itself through encoding/json +// (MarshalJSON and UnmarshalJSON, else MarshalText and UnmarshalText), or +// why it cannot: one half alone writes it one way and reads it another; a +// marshal method on the pointer alone is not called for the value a field +// holds; and a codec a struct gets from a field it embeds writes the whole +// struct as that field, dropping the rest. +func ownCodec(t types.Type) (bool, string) { + name := types.TypeString(t, func(p *types.Package) string { return p.Name() }) + for _, codec := range []string{"JSON", "Text"} { + marshal, onValue := methodOf(t, "Marshal"+codec) + _, unmarshal := methodOf(types.NewPointer(t), "Unmarshal"+codec) + if pointerOnly, _ := methodOf(types.NewPointer(t), "Marshal"+codec); pointerOnly != nil && !onValue { + return false, "holds a " + name + ", whose Marshal" + codec + " is on its pointer, which encoding/json does not call for a value" + } + switch { + case onValue && unmarshal: + if len(marshal.Index()) > 1 { + if st, ok := t.Underlying().(*types.Struct); ok && st.NumFields() > 1 { + return false, "holds a " + name + ", which takes Marshal" + codec + " from a field it embeds, so encoding/json writes that field alone and drops the rest" + } } + return true, "" + case onValue: + return false, "holds a " + name + ", which has Marshal" + codec + " but no Unmarshal" + codec + ", so encoding/json could not read it back" + case unmarshal: + return false, "holds a " + name + ", which has Unmarshal" + codec + " but no Marshal" + codec + ", so encoding/json would read it otherwise than it writes it" } } - return false + return false, "" +} + +// methodOf is the selection of t's method of that name, and whether t's +// method set has it. +func methodOf(t types.Type, name string) (*types.Selection, bool) { + ms := types.NewMethodSet(t) + for i := range ms.Len() { + if ms.At(i).Obj().Name() == name { + return ms.At(i), true + } + } + return nil, false } // notStorable says why the storage profile (vcffi.MarshalStorage and From 9d1df3497f8ba196bf54da4b844078c64b5e44f5 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 21:12:41 +1100 Subject: [PATCH 04/12] fix(go): refuse a document that embeds a pointer to an unexported struct encoding/json writes the fields such a struct promotes, but it cannot read them back: Unmarshal fails with "cannot set embedded pointer to unexported struct", since it cannot allocate one. stashgen accepted the shape, so every row encrypted and none decrypted. The walk now refuses it, naming the field; an embedded value of the same struct is still accepted, as encoding/json fills it in place. An embedded pointer to an exported struct round-trips, but its nil-ness is not stored: encoding/json allocates it when the document holds a member it promotes. That is documented in the stashgen README rather than refused, since a non-nil pointer with members written reads back as itself, and the shape is a common one. Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- languages/golang/CHANGELOG.md | 3 ++- languages/golang/cmd/stashgen/README.md | 2 ++ languages/golang/cmd/stashgen/main_test.go | 8 +++++++- languages/golang/stashgen/read.go | 18 +++++++++++++----- 4 files changed, 24 insertions(+), 7 deletions(-) diff --git a/languages/golang/CHANGELOG.md b/languages/golang/CHANGELOG.md index d9bba1d01..985516729 100644 --- a/languages/golang/CHANGELOG.md +++ b/languages/golang/CHANGELOG.md @@ -20,7 +20,8 @@ listed under **Breaking**. Releases are tagged `languages/golang/vX.Y.Z`. exactly. A pointer to one takes `null=sql`. `stashgen` refuses, when it runs, a type `encoding/json` cannot carry both ways: an interface such as `any`, a struct field `encoding/json` drops (unexported, tagged - `json:"-"`, or two writing one member), a one-way `MarshalJSON` or + `json:"-"`, or two writing one member), an embedded pointer to an + unexported struct (written, never read back), a one-way `MarshalJSON` or `MarshalText`, a channel, a function, a scalar, or a `[]byte`. The `Fields` entry's operands are Go values now: `Contains`, `ContainedBy`, `EqAt`, `Lt`, `Lte`, `Gt`, `Gte` and `Between` take `any`, diff --git a/languages/golang/cmd/stashgen/README.md b/languages/golang/cmd/stashgen/README.md index e0a7c7885..ddceed309 100644 --- a/languages/golang/cmd/stashgen/README.md +++ b/languages/golang/cmd/stashgen/README.md @@ -145,6 +145,8 @@ The generated code marshals a struct, map or slice type with `encoding/json` whe It calls `encoding/json` directly, never through `map[string]any`, so an `int64`, a `uint64` and a finite `float64` come back exactly. `stashgen` refuses a type that `encoding/json` cannot carry both ways: an interface such as `any` anywhere in it (it reads numbers back as `float64`); a struct field that `encoding/json` drops (unexported, tagged `json:"-"`, or one of two that write the same member); a `MarshalJSON` or `MarshalText` with no matching unmarshal method; a channel or a function. An unexported embedded struct is fine: its fields are promoted and stored. +An embedded pointer to an unexported struct is refused: `encoding/json` writes its fields but cannot read them back, since it cannot allocate the struct. +An embedded pointer to an exported struct is accepted, but whether it was nil is not stored: it comes back non-nil when the document holds a member it promotes, and nil when it holds none, so a non-nil pointer whose promoted fields are all left out by `omitempty` comes back nil. It refuses a string, a number and a bool too: a document is a struct, map or slice. A `[]byte` is refused, since `encoding/json` writes it as base64: declare JSON text as `json.RawMessage`. The engine parses the JSON text, so an object that names a member twice and an integer outside the 64-bit range are refused with `encrypt.ErrEncoding`. diff --git a/languages/golang/cmd/stashgen/main_test.go b/languages/golang/cmd/stashgen/main_test.go index 79621103d..d6aa4eed7 100644 --- a/languages/golang/cmd/stashgen/main_test.go +++ b/languages/golang/cmd/stashgen/main_test.go @@ -304,7 +304,7 @@ func TestRunGeneratesTypedJSONDocuments(t *testing.T) { t.Skip(err) } _ = checker.Close() - types := "type Profile struct {\n\tName string `json:\"name\"`\n\tAge int64 `json:\"age\"`\n\tVisits []Visit `json:\"visits\"`\n}\n\ntype Visit struct {\n\tAt int64 `json:\"at\"`\n}\n\ntype Prefs struct {\n\tTheme string `json:\"theme\"`\n}\n\ntype Hidden struct {\n\tName string\n\tsecret string\n}\n\ntype Loose struct {\n\tExtra any\n}\n\ntype Dropped struct {\n\tName string\n\tToken string `json:\"-\"`\n}\n\ntype Twice struct {\n\tName string `json:\"name\"`\n\tAlias string `json:\"name\"`\n}\n\ntype Label struct{ V string }\n\nfunc (l Label) MarshalText() ([]byte, error) { return []byte(l.V), nil }\n\ntype base struct {\n\tKind string `json:\"kind\"`\n}\n\ntype Embeds struct {\n\tbase\n\tName string `json:\"name\"`\n}\n\ntype Stamped struct {\n\ttime.Time\n\tName string\n}\n" + types := "type Profile struct {\n\tName string `json:\"name\"`\n\tAge int64 `json:\"age\"`\n\tVisits []Visit `json:\"visits\"`\n}\n\ntype Visit struct {\n\tAt int64 `json:\"at\"`\n}\n\ntype Prefs struct {\n\tTheme string `json:\"theme\"`\n}\n\ntype Hidden struct {\n\tName string\n\tsecret string\n}\n\ntype Loose struct {\n\tExtra any\n}\n\ntype Dropped struct {\n\tName string\n\tToken string `json:\"-\"`\n}\n\ntype Twice struct {\n\tName string `json:\"name\"`\n\tAlias string `json:\"name\"`\n}\n\ntype Label struct{ V string }\n\nfunc (l Label) MarshalText() ([]byte, error) { return []byte(l.V), nil }\n\ntype base struct {\n\tKind string `json:\"kind\"`\n}\n\ntype Embeds struct {\n\tbase\n\tName string `json:\"name\"`\n}\n\ntype PtrEmbeds struct {\n\t*base\n\tName string `json:\"name\"`\n}\n\ntype Shared struct {\n\tKind string `json:\"kind\"`\n}\n\ntype PtrShared struct {\n\t*Shared\n\tName string `json:\"name\"`\n}\n\ntype Stamped struct {\n\ttime.Time\n\tName string\n}\n" generate := func(fields string) (string, string, int) { source := "package users\n\nimport (\n\t\"encoding/json\"\n\t\"time\"\n)\n\nvar (\n\t_ json.RawMessage\n\t_ time.Time\n)\n\n" + types + "\ntype Customer struct {\n\t_ struct{} `stash:\"context=customers\"`\n\tID int64 `stash:\"id,passthrough\"`\n" + fields + "\n}\n" dir := writeModule(t, map[string]string{"model.go": source}) @@ -317,6 +317,11 @@ func TestRunGeneratesTypedJSONDocuments(t *testing.T) { if _, stderr, code := generate("\tProfile Embeds `stash:\"profile,encrypt_into=JsonSearch\"`"); code != 0 { t.Fatalf("an embedded unexported struct: exit %d\n%s", code, stderr) } + // encoding/json allocates an embedded pointer to an exported struct + // when it reads a member the struct promotes. + if _, stderr, code := generate("\tProfile PtrShared `stash:\"profile,encrypt_into=JsonSearch\"`"); code != 0 { + t.Fatalf("an embedded pointer to an exported struct: exit %d\n%s", code, stderr) + } written, stderr, code := generate("\tProfile Profile `stash:\"profile,encrypt_into=JsonSearch(ordered=$.age)\"`\n\tPrefs *Prefs `stash:\"prefs,encrypt_into=JsonSearch,null=sql\"`\n\tLabels map[string]string `stash:\"labels,encrypt_into=JsonSearch\"`\n\tScores []int64 `stash:\"scores,encrypt_into=JsonSearch\"`") if code != 0 { t.Fatalf("exit %d\n%s", code, stderr) @@ -365,6 +370,7 @@ func TestRunGeneratesTypedJSONDocuments(t *testing.T) { {"a field json drops", "\tProfile []Dropped `stash:\"profile,encrypt_into=JsonSearch\"`", "field Token tagged json:\"-\", which encoding/json drops"}, {"two fields, one member", "\tProfile Twice `stash:\"profile,encrypt_into=JsonSearch\"`", "fields Name and Alias both write the JSON member \"name\""}, {"a codec one way", "\tProfile map[string]Label `stash:\"profile,encrypt_into=JsonSearch\"`", "has MarshalText but no UnmarshalText"}, + {"an embedded pointer to an unexported struct", "\tProfile PtrEmbeds `stash:\"profile,encrypt_into=JsonSearch\"`", "embeds *base, a pointer to an unexported struct"}, {"a codec taken from an embedded field", "\tProfile Stamped `stash:\"profile,encrypt_into=JsonSearch\"`", "takes MarshalJSON from a field it embeds"}, {"a channel", "\tProfile []chan int `stash:\"profile,encrypt_into=JsonSearch\"`", "holds a chan int, which JSON does not carry"}, {"a pointer to a pointer", "\tProfile **Profile `stash:\"profile,encrypt_into=JsonSearch,null=sql\"`", "a pointer to a pointer"}, diff --git a/languages/golang/stashgen/read.go b/languages/golang/stashgen/read.go index b0e121793..c5c02a428 100644 --- a/languages/golang/stashgen/read.go +++ b/languages/golang/stashgen/read.go @@ -1250,9 +1250,11 @@ func notDocument(t types.Type) (string, bool) { // interface (read back as a map or a float64), a value JSON does not carry, // a map key encoding/json cannot write, and a struct field encoding/json // drops: an unexported one, one tagged json:"-", and two that write one -// member. A type with both halves of a codec (MarshalJSON and -// UnmarshalJSON, or MarshalText and UnmarshalText) writes and reads itself -// and is not looked into; one half alone is refused. +// member; and an embedded pointer to an unexported struct, whose promoted +// fields encoding/json writes and cannot read back. A type with both +// halves of a codec (MarshalJSON and UnmarshalJSON, or MarshalText and +// UnmarshalText) writes and reads itself and is not looked into; one half +// alone is refused. type documentWalk struct { seen map[types.Type]bool holdsStruct bool @@ -1310,13 +1312,19 @@ func (w *documentWalk) fields(u *types.Struct, names map[string]string, embeddin } if fld.Embedded() && name == "" { et := fld.Type() - if p, ok := et.Underlying().(*types.Pointer); ok { + p, embedsPointer := et.Underlying().(*types.Pointer) + if embedsPointer { et = p.Elem() } if st, ok := et.Underlying().(*types.Struct); ok { if own, _ := ownCodec(et); !own { // encoding/json promotes an untagged embedded - // struct's fields, exported or not the struct is. + // struct's fields, exported or not the struct is, + // but reading them through a nil pointer to an + // unexported struct fails: it cannot allocate one. + if embedsPointer && !fld.Exported() { + return "holds a struct that embeds *" + fld.Name() + ", a pointer to an unexported struct, whose fields encoding/json writes and cannot read back" + } if embedding[et] { continue } From 3ad7e5e25df1ba09bdce1e1d55ab95db304d08c3 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 21:13:32 +1100 Subject: [PATCH 05/12] fix(go): check a struct embedded through two branches each time The document walk remembered every struct it had embedded across the whole type, so a struct reached a second time through another branch (Doc{B; C}, each embedding A) was skipped. encoding/json writes neither of the two promoted members that then share a name, so Doc was stored as {} and both values were lost without an error. The walk now tracks embedded structs on the path it is descending only, which still stops an embedding cycle, and the duplicate-member check sees both promoted fields and refuses the type. Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- languages/golang/cmd/stashgen/main_test.go | 3 ++- languages/golang/stashgen/read.go | 9 +++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/languages/golang/cmd/stashgen/main_test.go b/languages/golang/cmd/stashgen/main_test.go index d6aa4eed7..7b146c34a 100644 --- a/languages/golang/cmd/stashgen/main_test.go +++ b/languages/golang/cmd/stashgen/main_test.go @@ -304,7 +304,7 @@ func TestRunGeneratesTypedJSONDocuments(t *testing.T) { t.Skip(err) } _ = checker.Close() - types := "type Profile struct {\n\tName string `json:\"name\"`\n\tAge int64 `json:\"age\"`\n\tVisits []Visit `json:\"visits\"`\n}\n\ntype Visit struct {\n\tAt int64 `json:\"at\"`\n}\n\ntype Prefs struct {\n\tTheme string `json:\"theme\"`\n}\n\ntype Hidden struct {\n\tName string\n\tsecret string\n}\n\ntype Loose struct {\n\tExtra any\n}\n\ntype Dropped struct {\n\tName string\n\tToken string `json:\"-\"`\n}\n\ntype Twice struct {\n\tName string `json:\"name\"`\n\tAlias string `json:\"name\"`\n}\n\ntype Label struct{ V string }\n\nfunc (l Label) MarshalText() ([]byte, error) { return []byte(l.V), nil }\n\ntype base struct {\n\tKind string `json:\"kind\"`\n}\n\ntype Embeds struct {\n\tbase\n\tName string `json:\"name\"`\n}\n\ntype PtrEmbeds struct {\n\t*base\n\tName string `json:\"name\"`\n}\n\ntype Shared struct {\n\tKind string `json:\"kind\"`\n}\n\ntype PtrShared struct {\n\t*Shared\n\tName string `json:\"name\"`\n}\n\ntype Stamped struct {\n\ttime.Time\n\tName string\n}\n" + types := "type Profile struct {\n\tName string `json:\"name\"`\n\tAge int64 `json:\"age\"`\n\tVisits []Visit `json:\"visits\"`\n}\n\ntype Visit struct {\n\tAt int64 `json:\"at\"`\n}\n\ntype Prefs struct {\n\tTheme string `json:\"theme\"`\n}\n\ntype Hidden struct {\n\tName string\n\tsecret string\n}\n\ntype Loose struct {\n\tExtra any\n}\n\ntype Dropped struct {\n\tName string\n\tToken string `json:\"-\"`\n}\n\ntype Twice struct {\n\tName string `json:\"name\"`\n\tAlias string `json:\"name\"`\n}\n\ntype Label struct{ V string }\n\nfunc (l Label) MarshalText() ([]byte, error) { return []byte(l.V), nil }\n\ntype base struct {\n\tKind string `json:\"kind\"`\n}\n\ntype Embeds struct {\n\tbase\n\tName string `json:\"name\"`\n}\n\ntype PtrEmbeds struct {\n\t*base\n\tName string `json:\"name\"`\n}\n\ntype Shared struct {\n\tKind string `json:\"kind\"`\n}\n\ntype PtrShared struct {\n\t*Shared\n\tName string `json:\"name\"`\n}\n\ntype Stamped struct {\n\ttime.Time\n\tName string\n}\n\ntype Inner struct {\n\tX int64\n}\n\ntype Left struct {\n\tInner\n}\n\ntype Right struct {\n\tInner\n}\n\ntype Diamond struct {\n\tLeft\n\tRight\n}\n" generate := func(fields string) (string, string, int) { source := "package users\n\nimport (\n\t\"encoding/json\"\n\t\"time\"\n)\n\nvar (\n\t_ json.RawMessage\n\t_ time.Time\n)\n\n" + types + "\ntype Customer struct {\n\t_ struct{} `stash:\"context=customers\"`\n\tID int64 `stash:\"id,passthrough\"`\n" + fields + "\n}\n" dir := writeModule(t, map[string]string{"model.go": source}) @@ -369,6 +369,7 @@ func TestRunGeneratesTypedJSONDocuments(t *testing.T) { {"an unexported field", "\tProfile Hidden `stash:\"profile,encrypt_into=JsonSearch\"`", "unexported field secret"}, {"a field json drops", "\tProfile []Dropped `stash:\"profile,encrypt_into=JsonSearch\"`", "field Token tagged json:\"-\", which encoding/json drops"}, {"two fields, one member", "\tProfile Twice `stash:\"profile,encrypt_into=JsonSearch\"`", "fields Name and Alias both write the JSON member \"name\""}, + {"one struct embedded through two branches", "\tProfile Diamond `stash:\"profile,encrypt_into=JsonSearch\"`", "fields X and X both write the JSON member \"X\""}, {"a codec one way", "\tProfile map[string]Label `stash:\"profile,encrypt_into=JsonSearch\"`", "has MarshalText but no UnmarshalText"}, {"an embedded pointer to an unexported struct", "\tProfile PtrEmbeds `stash:\"profile,encrypt_into=JsonSearch\"`", "embeds *base, a pointer to an unexported struct"}, {"a codec taken from an embedded field", "\tProfile Stamped `stash:\"profile,encrypt_into=JsonSearch\"`", "takes MarshalJSON from a field it embeds"}, diff --git a/languages/golang/stashgen/read.go b/languages/golang/stashgen/read.go index c5c02a428..431e8a804 100644 --- a/languages/golang/stashgen/read.go +++ b/languages/golang/stashgen/read.go @@ -1301,7 +1301,10 @@ func (w *documentWalk) problem(t types.Type) string { // fields checks a struct's fields, and those an untagged embedded struct // promotes into it, against names: the JSON member each field writes, so // two that write one member are refused (encoding/json keeps one of them, -// or neither, silently). +// or neither, silently). embedding holds the structs embedded on the path +// from the document's struct to u, and only those: it stops an embedding +// cycle, and a struct embedded through two branches is checked on each, +// so the members it promotes twice are refused. func (w *documentWalk) fields(u *types.Struct, names map[string]string, embedding map[types.Type]bool) string { for i := range u.NumFields() { fld := u.Field(i) @@ -1329,7 +1332,9 @@ func (w *documentWalk) fields(u *types.Struct, names map[string]string, embeddin continue } embedding[et] = true - if reason := w.fields(st, names, embedding); reason != "" { + reason := w.fields(st, names, embedding) + delete(embedding, et) + if reason != "" { return reason } continue From 2a811084ff3c9a0f271e7c3ba65f98ea6155c7aa Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 21:14:45 +1100 Subject: [PATCH 06/12] fix(go): look up a pointer field's codec on the pointer For a field of type *T the walk asked for UnmarshalJSON on **T, whose method set is empty, and refused a valid document as having MarshalJSON alone: *time.Time, *json.RawMessage, or a pointer to a type of the caller's with both methods. encoding/json calls the methods of *T on a pointer field when it writes it, and on the *T it allocates when it reads it, so both halves are now looked up on *T, whose method set holds T's methods too. A marshal method on the pointer alone is still refused for a value field, which encoding/json does not call it for. Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- languages/golang/cmd/stashgen/main_test.go | 7 ++- .../golang/encrypt/gensupport/json_test.go | 45 +++++++++++++++++++ languages/golang/stashgen/read.go | 25 ++++++++--- 3 files changed, 70 insertions(+), 7 deletions(-) diff --git a/languages/golang/cmd/stashgen/main_test.go b/languages/golang/cmd/stashgen/main_test.go index 7b146c34a..e856d0422 100644 --- a/languages/golang/cmd/stashgen/main_test.go +++ b/languages/golang/cmd/stashgen/main_test.go @@ -304,7 +304,7 @@ func TestRunGeneratesTypedJSONDocuments(t *testing.T) { t.Skip(err) } _ = checker.Close() - types := "type Profile struct {\n\tName string `json:\"name\"`\n\tAge int64 `json:\"age\"`\n\tVisits []Visit `json:\"visits\"`\n}\n\ntype Visit struct {\n\tAt int64 `json:\"at\"`\n}\n\ntype Prefs struct {\n\tTheme string `json:\"theme\"`\n}\n\ntype Hidden struct {\n\tName string\n\tsecret string\n}\n\ntype Loose struct {\n\tExtra any\n}\n\ntype Dropped struct {\n\tName string\n\tToken string `json:\"-\"`\n}\n\ntype Twice struct {\n\tName string `json:\"name\"`\n\tAlias string `json:\"name\"`\n}\n\ntype Label struct{ V string }\n\nfunc (l Label) MarshalText() ([]byte, error) { return []byte(l.V), nil }\n\ntype base struct {\n\tKind string `json:\"kind\"`\n}\n\ntype Embeds struct {\n\tbase\n\tName string `json:\"name\"`\n}\n\ntype PtrEmbeds struct {\n\t*base\n\tName string `json:\"name\"`\n}\n\ntype Shared struct {\n\tKind string `json:\"kind\"`\n}\n\ntype PtrShared struct {\n\t*Shared\n\tName string `json:\"name\"`\n}\n\ntype Stamped struct {\n\ttime.Time\n\tName string\n}\n\ntype Inner struct {\n\tX int64\n}\n\ntype Left struct {\n\tInner\n}\n\ntype Right struct {\n\tInner\n}\n\ntype Diamond struct {\n\tLeft\n\tRight\n}\n" + types := "type Profile struct {\n\tName string `json:\"name\"`\n\tAge int64 `json:\"age\"`\n\tVisits []Visit `json:\"visits\"`\n}\n\ntype Visit struct {\n\tAt int64 `json:\"at\"`\n}\n\ntype Prefs struct {\n\tTheme string `json:\"theme\"`\n}\n\ntype Hidden struct {\n\tName string\n\tsecret string\n}\n\ntype Loose struct {\n\tExtra any\n}\n\ntype Dropped struct {\n\tName string\n\tToken string `json:\"-\"`\n}\n\ntype Twice struct {\n\tName string `json:\"name\"`\n\tAlias string `json:\"name\"`\n}\n\ntype Label struct{ V string }\n\nfunc (l Label) MarshalText() ([]byte, error) { return []byte(l.V), nil }\n\ntype base struct {\n\tKind string `json:\"kind\"`\n}\n\ntype Embeds struct {\n\tbase\n\tName string `json:\"name\"`\n}\n\ntype PtrEmbeds struct {\n\t*base\n\tName string `json:\"name\"`\n}\n\ntype Shared struct {\n\tKind string `json:\"kind\"`\n}\n\ntype PtrShared struct {\n\t*Shared\n\tName string `json:\"name\"`\n}\n\ntype Stamped struct {\n\ttime.Time\n\tName string\n}\n\ntype Inner struct {\n\tX int64\n}\n\ntype Left struct {\n\tInner\n}\n\ntype Right struct {\n\tInner\n}\n\ntype Diamond struct {\n\tLeft\n\tRight\n}\n\ntype Code struct{ V string }\n\nfunc (c Code) MarshalJSON() ([]byte, error) { return json.Marshal(c.V) }\n\nfunc (c *Code) UnmarshalJSON(b []byte) error { return json.Unmarshal(b, &c.V) }\n\ntype Pointers struct {\n\tAt *time.Time `json:\"at\"`\n\tRaw *json.RawMessage `json:\"raw\"`\n\tCode *Code `json:\"code\"`\n}\n" generate := func(fields string) (string, string, int) { source := "package users\n\nimport (\n\t\"encoding/json\"\n\t\"time\"\n)\n\nvar (\n\t_ json.RawMessage\n\t_ time.Time\n)\n\n" + types + "\ntype Customer struct {\n\t_ struct{} `stash:\"context=customers\"`\n\tID int64 `stash:\"id,passthrough\"`\n" + fields + "\n}\n" dir := writeModule(t, map[string]string{"model.go": source}) @@ -317,6 +317,11 @@ func TestRunGeneratesTypedJSONDocuments(t *testing.T) { if _, stderr, code := generate("\tProfile Embeds `stash:\"profile,encrypt_into=JsonSearch\"`"); code != 0 { t.Fatalf("an embedded unexported struct: exit %d\n%s", code, stderr) } + // A pointer to a type with both halves of a codec writes and reads + // itself through the pointer's methods. + if _, stderr, code := generate("\tProfile Pointers `stash:\"profile,encrypt_into=JsonSearch\"`"); code != 0 { + t.Fatalf("pointers to codec types: exit %d\n%s", code, stderr) + } // encoding/json allocates an embedded pointer to an exported struct // when it reads a member the struct promotes. if _, stderr, code := generate("\tProfile PtrShared `stash:\"profile,encrypt_into=JsonSearch\"`"); code != 0 { diff --git a/languages/golang/encrypt/gensupport/json_test.go b/languages/golang/encrypt/gensupport/json_test.go index 7fe4ca768..402137d5d 100644 --- a/languages/golang/encrypt/gensupport/json_test.go +++ b/languages/golang/encrypt/gensupport/json_test.go @@ -6,6 +6,7 @@ import ( "math" "strings" "testing" + "time" "github.com/cipherstash/stack/languages/golang/encrypt" ) @@ -101,3 +102,47 @@ func TestGetDocumentReadsTheDeclaredType(t *testing.T) { t.Fatalf("a mismatched document: %v", err) } } + +// code writes and reads itself, MarshalJSON on the value and UnmarshalJSON +// on the pointer, as json.RawMessage and time.Time do. +type code struct{ v string } + +func (c code) MarshalJSON() ([]byte, error) { return json.Marshal("code:" + c.v) } + +func (c *code) UnmarshalJSON(b []byte) error { + var s string + if err := json.Unmarshal(b, &s); err != nil { + return err + } + c.v = strings.TrimPrefix(s, "code:") + return nil +} + +// pointers holds a pointer to each kind of codec type stashgen accepts in a +// document: the codec runs through the pointer both ways. +type pointers struct { + At *time.Time `json:"at"` + Raw *json.RawMessage `json:"raw"` + Code *code `json:"code"` + None *time.Time `json:"none"` +} + +func TestPointersToCodecTypesRoundTrip(t *testing.T) { + at := time.Date(2026, 10, 26, 10, 0, 0, 500, time.UTC) + raw := json.RawMessage(`{"a":[1,2]}`) + in := pointers{At: &at, Raw: &raw, Code: &code{v: "x"}} + text, err := JSONText(in) + if err != nil { + t.Fatal(err) + } + if want := `{"at":"2026-10-26T10:00:00.0000005Z","raw":{"a":[1,2]},"code":"code:x","none":null}`; string(text) != want { + t.Fatalf("JSONText = %s, want %s", text, want) + } + out, err := GetDocument[pointers](Values{"doc": text}, "doc") + if err != nil { + t.Fatal(err) + } + if out.At == nil || !out.At.Equal(at) || out.Raw == nil || string(*out.Raw) != string(raw) || out.Code == nil || out.Code.v != "x" || out.None != nil { + t.Fatalf("read back %+v", out) + } +} diff --git a/languages/golang/stashgen/read.go b/languages/golang/stashgen/read.go index 431e8a804..11c82ae0d 100644 --- a/languages/golang/stashgen/read.go +++ b/languages/golang/stashgen/read.go @@ -1364,23 +1364,36 @@ func (w *documentWalk) fields(u *types.Struct, names map[string]string, embeddin // marshal method on the pointer alone is not called for the value a field // holds; and a codec a struct gets from a field it embeds writes the whole // struct as that field, dropping the rest. +// +// A value's marshal method is looked up on its type, and its unmarshal +// method on a pointer to it, which encoding/json reads into. A pointer's +// are both looked up on the pointer: encoding/json writes a non-nil *E +// through *E's methods, and reads into the *E it allocates, and the method +// set of *E holds E's methods too. func ownCodec(t types.Type) (bool, string) { name := types.TypeString(t, func(p *types.Package) string { return p.Name() }) + elem, writer, reader := t, t, types.Type(types.NewPointer(t)) + p, isPointer := t.Underlying().(*types.Pointer) + if isPointer { + elem = p.Elem() + writer = types.NewPointer(elem) + reader = writer + } for _, codec := range []string{"JSON", "Text"} { - marshal, onValue := methodOf(t, "Marshal"+codec) - _, unmarshal := methodOf(types.NewPointer(t), "Unmarshal"+codec) - if pointerOnly, _ := methodOf(types.NewPointer(t), "Marshal"+codec); pointerOnly != nil && !onValue { + marshal, writes := methodOf(writer, "Marshal"+codec) + _, unmarshal := methodOf(reader, "Unmarshal"+codec) + if pointerOnly, _ := methodOf(reader, "Marshal"+codec); !isPointer && pointerOnly != nil && !writes { return false, "holds a " + name + ", whose Marshal" + codec + " is on its pointer, which encoding/json does not call for a value" } switch { - case onValue && unmarshal: + case writes && unmarshal: if len(marshal.Index()) > 1 { - if st, ok := t.Underlying().(*types.Struct); ok && st.NumFields() > 1 { + if st, ok := elem.Underlying().(*types.Struct); ok && st.NumFields() > 1 { return false, "holds a " + name + ", which takes Marshal" + codec + " from a field it embeds, so encoding/json writes that field alone and drops the rest" } } return true, "" - case onValue: + case writes: return false, "holds a " + name + ", which has Marshal" + codec + " but no Unmarshal" + codec + ", so encoding/json could not read it back" case unmarshal: return false, "holds a " + name + ", which has Unmarshal" + codec + " but no Marshal" + codec + ", so encoding/json would read it otherwise than it writes it" From 288b3ca7582d2ffc40b3d3b40ba259884167f82b Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 21:15:31 +1100 Subject: [PATCH 07/12] fix(go): refuse a json:",string" field in a document The walk cut the tag at its first comma and never saw the options, so `json:"age,string"` was accepted. encoding/json stores such a number or bool as a JSON string ("40"), so EqAt and every ordering at that path compare a number with a string leaf and match no row, with no error, and an ordered path orders the text. The option is refused on any field of a document, not only on an ordered path. It changes the JSON kind of the stored leaf, which breaks EqAt and containment by the number as much as an ordering, and it buys nothing here: a document's numbers already cross as their exact digits. Refusing it everywhere also needs no mapping from an ordered JSONPath back to the Go field it reaches. Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- languages/golang/CHANGELOG.md | 4 +++- languages/golang/cmd/stashgen/README.md | 1 + languages/golang/cmd/stashgen/main_test.go | 3 ++- languages/golang/stashgen/read.go | 25 +++++++++++++++------- 4 files changed, 23 insertions(+), 10 deletions(-) diff --git a/languages/golang/CHANGELOG.md b/languages/golang/CHANGELOG.md index 985516729..f031a34f7 100644 --- a/languages/golang/CHANGELOG.md +++ b/languages/golang/CHANGELOG.md @@ -20,7 +20,9 @@ listed under **Breaking**. Releases are tagged `languages/golang/vX.Y.Z`. exactly. A pointer to one takes `null=sql`. `stashgen` refuses, when it runs, a type `encoding/json` cannot carry both ways: an interface such as `any`, a struct field `encoding/json` drops (unexported, tagged - `json:"-"`, or two writing one member), an embedded pointer to an + `json:"-"`, or two writing one member), a field tagged + `json:",string"` (stored as a JSON string, which a query by its number + does not match), an embedded pointer to an unexported struct (written, never read back), a one-way `MarshalJSON` or `MarshalText`, a channel, a function, a scalar, or a `[]byte`. The `Fields` entry's operands are Go values now: `Contains`, `ContainedBy`, `EqAt`, `Lt`, `Lte`, `Gt`, `Gte` diff --git a/languages/golang/cmd/stashgen/README.md b/languages/golang/cmd/stashgen/README.md index ddceed309..658797467 100644 --- a/languages/golang/cmd/stashgen/README.md +++ b/languages/golang/cmd/stashgen/README.md @@ -144,6 +144,7 @@ people, err := customers.Decrypt(ctx, cipher, rows) // each Profile is a Profile The generated code marshals a struct, map or slice type with `encoding/json` when it seals the field, and unmarshals the opened document into the declared type. It calls `encoding/json` directly, never through `map[string]any`, so an `int64`, a `uint64` and a finite `float64` come back exactly. `stashgen` refuses a type that `encoding/json` cannot carry both ways: an interface such as `any` anywhere in it (it reads numbers back as `float64`); a struct field that `encoding/json` drops (unexported, tagged `json:"-"`, or one of two that write the same member); a `MarshalJSON` or `MarshalText` with no matching unmarshal method; a channel or a function. +It refuses a field tagged `json:",string"` anywhere in a document: `encoding/json` stores its number or bool as a JSON string (`"age":"40"`), so `EqAt`, an ordering or a needle by the number would match nothing, and an ordered path would order the text. A document's numbers cross as their exact digits without it. An unexported embedded struct is fine: its fields are promoted and stored. An embedded pointer to an unexported struct is refused: `encoding/json` writes its fields but cannot read them back, since it cannot allocate the struct. An embedded pointer to an exported struct is accepted, but whether it was nil is not stored: it comes back non-nil when the document holds a member it promotes, and nil when it holds none, so a non-nil pointer whose promoted fields are all left out by `omitempty` comes back nil. diff --git a/languages/golang/cmd/stashgen/main_test.go b/languages/golang/cmd/stashgen/main_test.go index e856d0422..e315fa3c9 100644 --- a/languages/golang/cmd/stashgen/main_test.go +++ b/languages/golang/cmd/stashgen/main_test.go @@ -304,7 +304,7 @@ func TestRunGeneratesTypedJSONDocuments(t *testing.T) { t.Skip(err) } _ = checker.Close() - types := "type Profile struct {\n\tName string `json:\"name\"`\n\tAge int64 `json:\"age\"`\n\tVisits []Visit `json:\"visits\"`\n}\n\ntype Visit struct {\n\tAt int64 `json:\"at\"`\n}\n\ntype Prefs struct {\n\tTheme string `json:\"theme\"`\n}\n\ntype Hidden struct {\n\tName string\n\tsecret string\n}\n\ntype Loose struct {\n\tExtra any\n}\n\ntype Dropped struct {\n\tName string\n\tToken string `json:\"-\"`\n}\n\ntype Twice struct {\n\tName string `json:\"name\"`\n\tAlias string `json:\"name\"`\n}\n\ntype Label struct{ V string }\n\nfunc (l Label) MarshalText() ([]byte, error) { return []byte(l.V), nil }\n\ntype base struct {\n\tKind string `json:\"kind\"`\n}\n\ntype Embeds struct {\n\tbase\n\tName string `json:\"name\"`\n}\n\ntype PtrEmbeds struct {\n\t*base\n\tName string `json:\"name\"`\n}\n\ntype Shared struct {\n\tKind string `json:\"kind\"`\n}\n\ntype PtrShared struct {\n\t*Shared\n\tName string `json:\"name\"`\n}\n\ntype Stamped struct {\n\ttime.Time\n\tName string\n}\n\ntype Inner struct {\n\tX int64\n}\n\ntype Left struct {\n\tInner\n}\n\ntype Right struct {\n\tInner\n}\n\ntype Diamond struct {\n\tLeft\n\tRight\n}\n\ntype Code struct{ V string }\n\nfunc (c Code) MarshalJSON() ([]byte, error) { return json.Marshal(c.V) }\n\nfunc (c *Code) UnmarshalJSON(b []byte) error { return json.Unmarshal(b, &c.V) }\n\ntype Pointers struct {\n\tAt *time.Time `json:\"at\"`\n\tRaw *json.RawMessage `json:\"raw\"`\n\tCode *Code `json:\"code\"`\n}\n" + types := "type Profile struct {\n\tName string `json:\"name\"`\n\tAge int64 `json:\"age\"`\n\tVisits []Visit `json:\"visits\"`\n}\n\ntype Visit struct {\n\tAt int64 `json:\"at\"`\n}\n\ntype Prefs struct {\n\tTheme string `json:\"theme\"`\n}\n\ntype Hidden struct {\n\tName string\n\tsecret string\n}\n\ntype Loose struct {\n\tExtra any\n}\n\ntype Dropped struct {\n\tName string\n\tToken string `json:\"-\"`\n}\n\ntype Twice struct {\n\tName string `json:\"name\"`\n\tAlias string `json:\"name\"`\n}\n\ntype Label struct{ V string }\n\nfunc (l Label) MarshalText() ([]byte, error) { return []byte(l.V), nil }\n\ntype base struct {\n\tKind string `json:\"kind\"`\n}\n\ntype Embeds struct {\n\tbase\n\tName string `json:\"name\"`\n}\n\ntype PtrEmbeds struct {\n\t*base\n\tName string `json:\"name\"`\n}\n\ntype Shared struct {\n\tKind string `json:\"kind\"`\n}\n\ntype PtrShared struct {\n\t*Shared\n\tName string `json:\"name\"`\n}\n\ntype Stamped struct {\n\ttime.Time\n\tName string\n}\n\ntype Inner struct {\n\tX int64\n}\n\ntype Left struct {\n\tInner\n}\n\ntype Right struct {\n\tInner\n}\n\ntype Diamond struct {\n\tLeft\n\tRight\n}\n\ntype Code struct{ V string }\n\nfunc (c Code) MarshalJSON() ([]byte, error) { return json.Marshal(c.V) }\n\nfunc (c *Code) UnmarshalJSON(b []byte) error { return json.Unmarshal(b, &c.V) }\n\ntype Pointers struct {\n\tAt *time.Time `json:\"at\"`\n\tRaw *json.RawMessage `json:\"raw\"`\n\tCode *Code `json:\"code\"`\n}\n\ntype Quoted struct {\n\tAge int64 `json:\"age,omitempty,string\"`\n}\n" generate := func(fields string) (string, string, int) { source := "package users\n\nimport (\n\t\"encoding/json\"\n\t\"time\"\n)\n\nvar (\n\t_ json.RawMessage\n\t_ time.Time\n)\n\n" + types + "\ntype Customer struct {\n\t_ struct{} `stash:\"context=customers\"`\n\tID int64 `stash:\"id,passthrough\"`\n" + fields + "\n}\n" dir := writeModule(t, map[string]string{"model.go": source}) @@ -375,6 +375,7 @@ func TestRunGeneratesTypedJSONDocuments(t *testing.T) { {"a field json drops", "\tProfile []Dropped `stash:\"profile,encrypt_into=JsonSearch\"`", "field Token tagged json:\"-\", which encoding/json drops"}, {"two fields, one member", "\tProfile Twice `stash:\"profile,encrypt_into=JsonSearch\"`", "fields Name and Alias both write the JSON member \"name\""}, {"one struct embedded through two branches", "\tProfile Diamond `stash:\"profile,encrypt_into=JsonSearch\"`", "fields X and X both write the JSON member \"X\""}, + {"a number stored as a string", "\tProfile Quoted `stash:\"profile,encrypt_into=JsonSearch(ordered=$.age)\"`", "field Age tagged json:\",string\""}, {"a codec one way", "\tProfile map[string]Label `stash:\"profile,encrypt_into=JsonSearch\"`", "has MarshalText but no UnmarshalText"}, {"an embedded pointer to an unexported struct", "\tProfile PtrEmbeds `stash:\"profile,encrypt_into=JsonSearch\"`", "embeds *base, a pointer to an unexported struct"}, {"a codec taken from an embedded field", "\tProfile Stamped `stash:\"profile,encrypt_into=JsonSearch\"`", "takes MarshalJSON from a field it embeds"}, diff --git a/languages/golang/stashgen/read.go b/languages/golang/stashgen/read.go index 11c82ae0d..7ad8cd2f5 100644 --- a/languages/golang/stashgen/read.go +++ b/languages/golang/stashgen/read.go @@ -1248,13 +1248,14 @@ func notDocument(t types.Type) (string, bool) { // it, once per type: what it writes must read back to the same value, so // nothing is dropped and nothing changes type on the way. It refuses an // interface (read back as a map or a float64), a value JSON does not carry, -// a map key encoding/json cannot write, and a struct field encoding/json -// drops: an unexported one, one tagged json:"-", and two that write one -// member; and an embedded pointer to an unexported struct, whose promoted -// fields encoding/json writes and cannot read back. A type with both -// halves of a codec (MarshalJSON and UnmarshalJSON, or MarshalText and -// UnmarshalText) writes and reads itself and is not looked into; one half -// alone is refused. +// a map key encoding/json cannot write, a struct field encoding/json drops +// (an unexported one, one tagged json:"-", two that write one member), a +// field tagged json:",string", stored as a JSON string that a query by its +// number or bool does not match, and an embedded pointer to an unexported +// struct, whose promoted fields encoding/json writes and cannot read +// back. A type with both halves of a codec (MarshalJSON and UnmarshalJSON, +// or MarshalText and UnmarshalText) writes and reads itself and is not +// looked into; one half alone is refused. type documentWalk struct { seen map[types.Type]bool holdsStruct bool @@ -1309,10 +1310,18 @@ func (w *documentWalk) fields(u *types.Struct, names map[string]string, embeddin for i := range u.NumFields() { fld := u.Field(i) tag := reflect.StructTag(u.Tag(i)).Get("json") - name, _, _ := strings.Cut(tag, ",") + name, options, _ := strings.Cut(tag, ",") if tag == "-" { return "holds a struct with a field " + fld.Name() + ` tagged json:"-", which encoding/json drops` } + // ",string" writes a number or bool as a JSON string, so a query + // by the number or bool at that path matches nothing. A + // document's numbers cross as their exact digits without it. + for option := range strings.SplitSeq(options, ",") { + if option == "string" { + return "holds a struct with a field " + fld.Name() + ` tagged json:",string", which encoding/json stores as a JSON string, so EqAt, an ordering or a needle by its number or bool would match nothing` + } + } if fld.Embedded() && name == "" { et := fld.Type() p, embedsPointer := et.Underlying().(*types.Pointer) From a32b09c864b95bfed8cc17f8716fd632feda7045 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 21:16:34 +1100 Subject: [PATCH 08/12] docs(go): say what a document of a Go type does not keep The READMEs, the package docs and the CHANGELOG said a document comes back exactly. That holds for numbers only. encoding/json writes invalid UTF-8 in a string as U+FFFD, with no error; a nil map or slice is stored as JSON null, which a containment query with an empty one does not match; and a time.Time inside a document is stored as RFC 3339 text in the caller's offset with trailing zero nanoseconds trimmed, so one instant in two zones is two strings, and an ordering on that path compares text, not time. All three are documented rather than refused. Invalid UTF-8 is a property of a value, not a type: catching it would need a reflective walk of the caller's value at run time, which the SDK principles keep out of the SDK and generated code (Go principle 4). The time case is not limited to ordered paths (EqAt and containment miss across zones too), mapping an ordered JSONPath back to the Go field it reaches would need a second walk copying encoding/json's naming rules, and a codec whose text sorts in value order (a fixed-width UTC timestamp) is legitimate. The README now also says that only a time.Time document field is refused, not a time inside one. Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- languages/golang/CHANGELOG.md | 6 +++++- languages/golang/cmd/stashgen/README.md | 6 ++++++ languages/golang/encrypt/README.md | 4 +++- languages/golang/encrypt/doc.go | 10 +++++++++- 4 files changed, 23 insertions(+), 3 deletions(-) diff --git a/languages/golang/CHANGELOG.md b/languages/golang/CHANGELOG.md index f031a34f7..69ffd0330 100644 --- a/languages/golang/CHANGELOG.md +++ b/languages/golang/CHANGELOG.md @@ -17,7 +17,11 @@ listed under **Breaking**. Releases are tagged `languages/golang/vX.Y.Z`. marshals it with `encoding/json` when it seals it, and reads the opened document back into the declared type (`gensupport.GetDocument`, `GetDocumentPtr`), so 64-bit integers and finite `float64`s round-trip - exactly. A pointer to one takes `null=sql`. `stashgen` refuses, when it + exactly. A pointer to one takes `null=sql`. The rest of a document comes + back as `encoding/json` reads it: invalid UTF-8 in a string becomes + U+FFFD, a nil map or slice is stored as JSON `null` (not `{}` or `[]`), + and a `time.Time` inside a document is stored as RFC 3339 text in its + own offset, which `EqAt` and the orderings compare as text. `stashgen` refuses, when it runs, a type `encoding/json` cannot carry both ways: an interface such as `any`, a struct field `encoding/json` drops (unexported, tagged `json:"-"`, or two writing one member), a field tagged diff --git a/languages/golang/cmd/stashgen/README.md b/languages/golang/cmd/stashgen/README.md index 658797467..d9c01b037 100644 --- a/languages/golang/cmd/stashgen/README.md +++ b/languages/golang/cmd/stashgen/README.md @@ -143,6 +143,12 @@ people, err := customers.Decrypt(ctx, cipher, rows) // each Profile is a Profile The generated code marshals a struct, map or slice type with `encoding/json` when it seals the field, and unmarshals the opened document into the declared type. It calls `encoding/json` directly, never through `map[string]any`, so an `int64`, a `uint64` and a finite `float64` come back exactly. +The rest of the document comes back as `encoding/json` reads what it wrote, which is not always the value sealed: + +- Invalid UTF-8 in a Go string is written as U+FFFD, with no error: `"a\xffb"` comes back as `"a\uFFFDb"`, the replacement character in place of the byte. `stashgen` cannot refuse it, because it is a property of a value, not of a type, and finding it would need a reflective walk of each value at run time, which the generated code does not do. Check text that may not be UTF-8 with `utf8.ValidString` before you seal it. +- A nil map or slice is stored as JSON `null`, and an empty one as `{}` or `[]`. Each reads back as it was (a field with `null=sql` stores a nil pointer as SQL `NULL` instead), but a query tells them apart: `Contains` with an empty map does not match a stored `null`. +- A `time.Time` field is refused, since it is one value, not a document, but a `time.Time` inside a document is accepted. It is stored as the text its `MarshalJSON` writes: RFC 3339, in the time's own offset, with trailing zero nanoseconds left out. One instant in two zones is two strings, so `EqAt` with a time in another zone matches nothing, and an ordering on that path orders the text, which is not time order: `…T10:00:00+10:00` sorts after `…T01:00:00Z`, an earlier instant, and `…:00.5Z` sorts before `…:00Z`. If you query a time, store it as an integer (`t.UnixNano()`) or as UTC text of a fixed width. Any type with its own `MarshalJSON` or `MarshalText` is stored as the text it writes, and queried as that text. + `stashgen` refuses a type that `encoding/json` cannot carry both ways: an interface such as `any` anywhere in it (it reads numbers back as `float64`); a struct field that `encoding/json` drops (unexported, tagged `json:"-"`, or one of two that write the same member); a `MarshalJSON` or `MarshalText` with no matching unmarshal method; a channel or a function. It refuses a field tagged `json:",string"` anywhere in a document: `encoding/json` stores its number or bool as a JSON string (`"age":"40"`), so `EqAt`, an ordering or a needle by the number would match nothing, and an ordered path would order the text. A document's numbers cross as their exact digits without it. An unexported embedded struct is fine: its fields are promoted and stored. diff --git a/languages/golang/encrypt/README.md b/languages/golang/encrypt/README.md index 7b32b4766..fce38c1cb 100644 --- a/languages/golang/encrypt/README.md +++ b/languages/golang/encrypt/README.md @@ -597,7 +597,9 @@ column. column, from a struct, map or slice field, or a `json.RawMessage` (`stash:"profile,encrypt_into=JsonSearch(ordered=$.age)"`; see the stashgen README). A struct is marshalled with `encoding/json` when it is -sealed and read back into its type when it is opened. One entry per node is +sealed and read back into its type when it is opened. Its numbers come back +exactly; the stashgen README lists what `encoding/json` does not keep +(invalid UTF-8, a nil map or slice, a time's zone). One entry per node is encrypted, so the database can test containment, equality and ordering at a path, return one node with `->`, and walk a subtree, without decrypting: diff --git a/languages/golang/encrypt/doc.go b/languages/golang/encrypt/doc.go index e861ec229..97d2b307f 100644 --- a/languages/golang/encrypt/doc.go +++ b/languages/golang/encrypt/doc.go @@ -207,7 +207,15 @@ // text, or as a struct, map or slice type. The generated code marshals such // a type with encoding/json when it seals the field. It unmarshals the // opened text into the declared type, so integers to 64 bits and finite -// float64s come back exactly. +// float64s come back exactly. The rest comes back as encoding/json reads +// what it wrote, which is not always the value sealed: invalid UTF-8 in a +// string becomes U+FFFD, with no error (a property of the value, which the +// generated code does not walk, so check it with utf8.ValidString first); a +// nil map or slice is stored as JSON null, which a needle of an empty one +// does not match; and a time.Time inside a document is stored as RFC 3339 +// text in its own offset, so EqAt with a time in another zone misses and +// an ordering on that path orders the text, not the instants. Store a time +// you query as an integer, such as t.UnixNano(). // // The field's operands are Go values. Contains and ContainedBy take a // needle, and EqAt and the orderings take a scalar. Each is a From 93301342a0463feac64f65d39132d42090cd8617 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 21:20:33 +1100 Subject: [PATCH 09/12] fix(go): refuse a pointer-to-string or null containment needle The string-needle guard looked at the needle's dynamic kind only, so a *string or *json.Number passed and was written as one JSON string, the mistake the guard exists to stop. It now looks through one pointer, as encoding/json does when it writes one, using the type alone (no new reflection on the value). The same look-through applies to the []byte guard in JSONText, so a *[]byte is refused rather than written as base64. Contains and ContainedBy also refuse a needle that writes JSON null: nil, or a nil pointer, map or slice. As a needle it matches only a document that is itself null, which is almost never what a caller who passed nil meant. The check reads the marshalled text, so it needs no reflection. The generated Contains comment now says the own-type refusal covers a slice or map of structs too ([]Visit), and names the form to use instead: a needle type with pointer and omitempty fields, used where the field's type holds a struct ([]VisitNeedle). JSONText's comment states the []byte rule exactly: a slice of a byte-kind element is refused even when the element writes itself, which is safe and needs no look at the element's methods. Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- languages/golang/CHANGELOG.md | 5 +- languages/golang/cmd/stashgen/README.md | 6 +- languages/golang/cmd/stashgen/main_test.go | 6 +- languages/golang/encrypt/doc.go | 10 +++- languages/golang/encrypt/gensupport/json.go | 53 +++++++++++++----- .../golang/encrypt/gensupport/json_test.go | 38 +++++++++++++ .../internal/testusers/customer_stash.go | 32 ++++++----- .../internal/testusers/subscriber_stash.go | 56 +++++++++++-------- languages/golang/stashgen/emit.go | 11 ++-- languages/golang/stashgen/read.go | 9 ++- 10 files changed, 159 insertions(+), 67 deletions(-) diff --git a/languages/golang/CHANGELOG.md b/languages/golang/CHANGELOG.md index 69ffd0330..a744d737c 100644 --- a/languages/golang/CHANGELOG.md +++ b/languages/golang/CHANGELOG.md @@ -32,8 +32,9 @@ listed under **Breaking**. Releases are tagged `languages/golang/vX.Y.Z`. values now: `Contains`, `ContainedBy`, `EqAt`, `Lt`, `Lte`, `Gt`, `Gte` and `Between` take `any`, a `json.RawMessage` crossing as the JSON text it is and anything else - marshalled (`gensupport.JSONText`); a `[]byte` or a type defined over - one is refused, and so is a Go string as a containment needle. A zero + marshalled (`gensupport.JSONText`); a `[]byte`, a type defined over + one, or a pointer to either is refused, and so is a containment needle + that is a Go string, a pointer to one, or writes JSON `null`. A zero struct field without `omitempty` narrows a containment query, so `Contains` on a field whose type holds a struct refuses a needle of that type (`gensupport.NewDocumentField`); the generated doc comments say how diff --git a/languages/golang/cmd/stashgen/README.md b/languages/golang/cmd/stashgen/README.md index d9c01b037..28c4f619e 100644 --- a/languages/golang/cmd/stashgen/README.md +++ b/languages/golang/cmd/stashgen/README.md @@ -182,13 +182,15 @@ An ordering compares the value `->` extracts at its path, and EQL's `<` compares `OpenOne` reads one value and refuses a read whose path has a wildcard before anything is opened; open a wildcard read with `Open`. A needle and a value are Go values: a `json.RawMessage` of JSON text crosses as it is, and anything else is marshalled with `encoding/json`. `EqAt` and the orderings take a scalar: a string, a number, a bool, or `nil` for JSON `null`. -A `[]byte`, or a type defined over one, is refused rather than written as base64. -A Go string is refused as a `Contains` or `ContainedBy` needle, since it would be one JSON string: pass JSON text as `json.RawMessage`. +A `[]byte`, a type defined over one, or a pointer to either, is refused rather than written as base64. +A Go string, or a pointer to one, is refused as a `Contains` or `ContainedBy` needle, since it would be one JSON string: pass JSON text as `json.RawMessage`. +So is a needle that writes JSON `null` (`nil`, or a nil pointer, map or slice), which only a null document contains. For `EqAt` and the orderings a Go string is the string value itself. A document contains a needle when it holds every member the needle writes, and `encoding/json` writes a struct field that has no `omitempty` even when it is zero. So a zero field narrows the query to the documents that hold that zero: `Profile{Name: "Alice"}` also asks for `"age": 0`. Give a needle struct pointer fields with `omitempty`, as `ProfileNeedle` above, or use a map. `Contains` refuses a needle of the field's own type when that type holds a struct, since that is the likeliest way to make the mistake; a type defined over it (`type wholeProfile Profile`) states that you mean every field. +That includes a slice or map of structs: a `[]Visit` document refuses a `[]Visit` needle, since each `Visit` writes its zero fields; pass a `[]VisitNeedle`, a slice of a needle type with pointer fields and `omitempty`. `ContainedBy` takes the field's own type: there a zero field widens the query. Each value a selection or read opens is JSON text; `encrypt.DecodeJSON[T]` reads one into a Go type, with the same rules as a whole document. A position in a path is refused for `EqAt` and `Selector`, whose selector reads every position as the wildcard; read a position with `Subtree`. diff --git a/languages/golang/cmd/stashgen/main_test.go b/languages/golang/cmd/stashgen/main_test.go index e315fa3c9..41cc29cd9 100644 --- a/languages/golang/cmd/stashgen/main_test.go +++ b/languages/golang/cmd/stashgen/main_test.go @@ -327,7 +327,7 @@ func TestRunGeneratesTypedJSONDocuments(t *testing.T) { if _, stderr, code := generate("\tProfile PtrShared `stash:\"profile,encrypt_into=JsonSearch\"`"); code != 0 { t.Fatalf("an embedded pointer to an exported struct: exit %d\n%s", code, stderr) } - written, stderr, code := generate("\tProfile Profile `stash:\"profile,encrypt_into=JsonSearch(ordered=$.age)\"`\n\tPrefs *Prefs `stash:\"prefs,encrypt_into=JsonSearch,null=sql\"`\n\tLabels map[string]string `stash:\"labels,encrypt_into=JsonSearch\"`\n\tScores []int64 `stash:\"scores,encrypt_into=JsonSearch\"`") + written, stderr, code := generate("\tProfile Profile `stash:\"profile,encrypt_into=JsonSearch(ordered=$.age)\"`\n\tPrefs *Prefs `stash:\"prefs,encrypt_into=JsonSearch,null=sql\"`\n\tLabels map[string]string `stash:\"labels,encrypt_into=JsonSearch\"`\n\tScores []int64 `stash:\"scores,encrypt_into=JsonSearch\"`\n\tVisits []Visit `stash:\"visits,encrypt_into=JsonSearch\"`") if code != 0 { t.Fatalf("exit %d\n%s", code, stderr) } @@ -351,7 +351,9 @@ func TestRunGeneratesTypedJSONDocuments(t *testing.T) { `Scores: newScoresField(gensupport.NewField[[]int64](declaration, "scores")),`, "func (f ProfileField) Encrypt(ctx context.Context, c *encrypt.Cipher, v Profile) (eql.JSONSearch, error)", "func (f ProfileField) Contains(ctx context.Context, c *encrypt.Cipher, needle any) (eql.JSONSearchQuery, error)", - "A needle of the field's own type, Profile, is refused for that reason.", + "A needle of the field's own type, Profile, is refused for that reason, and so is a pointer to it: write a needle type", + // A slice of structs refuses itself too, and says what to pass. + "A needle of the field's own type, []Visit, is refused for that reason, and so is a pointer to it: each struct it holds writes its zero fields too.", "func (p ProfileFieldPath) Gt(ctx context.Context, c *encrypt.Cipher, v any) (eql.JSONSearchLeafQuery, error)", } { // A doc comment is wrapped: read it as one line. diff --git a/languages/golang/encrypt/doc.go b/languages/golang/encrypt/doc.go index 97d2b307f..903b455b9 100644 --- a/languages/golang/encrypt/doc.go +++ b/languages/golang/encrypt/doc.go @@ -225,15 +225,19 @@ // older, err := customers.Fields.Profile.At.Age.Gt(ctx, cipher, 40) // // A Go string is the string value itself, never parsed. So Contains and -// ContainedBy refuse a string needle; put JSON text in a json.RawMessage. +// ContainedBy refuse a string needle, or a pointer to one; put JSON text in +// a json.RawMessage. They refuse a needle that writes JSON null (nil, or a +// nil pointer, map or slice) too: only a null document contains it. // // A document contains a needle when it holds every member the needle writes. // encoding/json writes a struct field that has no omitempty even when the // field is zero. So a zero field narrows the query to documents that hold // that zero. Give a needle struct pointer fields with omitempty, or use a // map. Contains refuses a needle of the field's own type when that type -// holds a struct, the likeliest way to make this mistake. ContainedBy takes -// one: there a zero field widens the query. +// holds a struct, the likeliest way to make this mistake, a slice of +// structs included: for a []Visit document, pass a slice of a needle type +// with pointer fields. ContainedBy takes one: there a zero field widens +// the query. // // A selection or a subtree read opens values as JSON text ([JSONValue], // [JSONRead.Open]). [DecodeJSON] reads one into a Go type. diff --git a/languages/golang/encrypt/gensupport/json.go b/languages/golang/encrypt/gensupport/json.go index df23ac51d..488ce8198 100644 --- a/languages/golang/encrypt/gensupport/json.go +++ b/languages/golang/encrypt/gensupport/json.go @@ -1,6 +1,7 @@ package gensupport import ( + "bytes" "context" "encoding" "encoding/json" @@ -28,6 +29,13 @@ import ( // write it as a base64 string, which is never what JSON text in a []byte // means; convert it, json.RawMessage(b). // +// The []byte rule is by kind: a value of a type whose kind is a slice of a +// byte-kind element, or a pointer to one, is refused unless the value's +// own type writes itself (json.Marshaler or encoding.TextMarshaler). So a +// []B is refused even when B or *B has MarshalJSON, though encoding/json +// would then write an array: refusing it is safe, and telling the two +// apart would mean asking for the element's methods. +// // An error wraps encrypt.ErrEncoding and names a Go type, never the value: // encoding/json's own text quotes a NaN, and a MarshalJSON's error may hold // anything. @@ -35,9 +43,11 @@ func JSONText(v any) ([]byte, error) { if raw, ok := v.(json.RawMessage); ok { return raw, nil } - if writesItself(v) == "" && kindOf(v) == reflect.Slice && reflect.TypeOf(v).Elem().Kind() == reflect.Uint8 { + if t := throughPointer(v); writesItself(v) == "" && t != nil && t.Kind() == reflect.Slice && t.Elem().Kind() == reflect.Uint8 { // A []byte, or a type defined over one (type Doc []byte, or - // json.RawMessage redefined, which drops its MarshalJSON). + // json.RawMessage redefined, which drops its MarshalJSON), or a + // pointer to either, which encoding/json writes as what it + // points to. return nil, fmt.Errorf("%w: a %T is not JSON text to encoding/json, which writes it as a base64 string; convert JSON text to json.RawMessage", encrypt.ErrEncoding, v) } text, err := json.Marshal(v) @@ -70,15 +80,18 @@ func writesItself(v any) string { return "" } -// kindOf is the kind of v's dynamic type, or reflect.Invalid for nil: the -// one place the JSON operands look at a type by kind, since a type switch -// cannot name every type defined over []byte or string. -func kindOf(v any) reflect.Kind { +// throughPointer is v's dynamic type, or what it points to when it is a +// pointer, or nil for nil: the one place the JSON operands look at a type +// by kind, since a type switch cannot name every type defined over []byte +// or string. encoding/json writes a non-nil pointer as the value it points +// to, so a *string is one JSON string as a string is. Only the type is +// read, never the value. +func throughPointer(v any) reflect.Type { t := reflect.TypeOf(v) - if t == nil { - return reflect.Invalid + if t != nil && t.Kind() == reflect.Pointer { + return t.Elem() } - return t.Kind() + return t } // GetDocument reads one opened document field into the Go type T the struct @@ -156,17 +169,22 @@ func (f Field[T]) jsonOperand(v any) ([]byte, error) { // JSONQuery derives a containment operator's query on a JSON field // (contains, contained_by): the needle's projection, Q. The needle is any -// value [JSONText] writes, other than a string, which is refused: one JSON -// string is never what a containment needle means, and JSON text in a Go -// string is the likely mistake. Contains on a field of [NewDocumentField] -// refuses a needle of the field's own type, T or *T: its zero-valued -// fields would narrow the query. +// value [JSONText] writes, other than a string or a pointer to one, which +// is refused: one JSON string is never what a containment needle means, +// and JSON text in a Go string is the likely mistake. A needle that writes +// JSON null (nil, or a nil pointer, map or slice) is refused too: it +// matches only a document that is itself null. Contains on a field of +// [NewDocumentField] refuses a needle of the field's own type, T or *T: +// its zero-valued fields would narrow the query. func JSONQuery[Q ~[]byte, T any](ctx context.Context, f Field[T], c *encrypt.Cipher, operator string, needle any) (Q, error) { // A Go string is one JSON string, and a document of members never // contains one: it is almost always JSON text meant as the needle. - if kindOf(needle) == reflect.String && writesItself(needle) == "" { + if t := throughPointer(needle); t != nil && t.Kind() == reflect.String && writesItself(needle) == "" { return nil, fmt.Errorf("%w: gensupport: %s: a %T needle is one JSON string, not JSON text; pass JSON text as json.RawMessage, or a map or struct", encrypt.ErrEncoding, f.name, needle) } + if needle == nil { + return nil, fmt.Errorf("%w: gensupport: %s: a nil needle is JSON null, which only a null document contains; pass a map, a struct or JSON text", encrypt.ErrEncoding, f.name) + } if f.holdsStruct && operator == "contains" { switch needle.(type) { case T, *T: @@ -177,6 +195,11 @@ func JSONQuery[Q ~[]byte, T any](ctx context.Context, f Field[T], c *encrypt.Cip if err != nil { return nil, err } + // A nil pointer, map or slice: the text says so without a look at the + // value. + if bytes.Equal(text, []byte("null")) { + return nil, fmt.Errorf("%w: gensupport: %s: a %T needle that writes JSON null (a nil one) matches only a null document; pass a map, a struct or JSON text", encrypt.ErrEncoding, f.name, needle) + } term, err := f.jsonTerm(ctx, c, operator, "eql", 1, vcvalue.Object{{Key: "json", Value: text}}) if err != nil { return nil, err diff --git a/languages/golang/encrypt/gensupport/json_test.go b/languages/golang/encrypt/gensupport/json_test.go index 402137d5d..41b5ff05e 100644 --- a/languages/golang/encrypt/gensupport/json_test.go +++ b/languages/golang/encrypt/gensupport/json_test.go @@ -1,6 +1,7 @@ package gensupport import ( + "context" "encoding/json" "errors" "math" @@ -146,3 +147,40 @@ func TestPointersToCodecTypesRoundTrip(t *testing.T) { t.Fatalf("read back %+v", out) } } + +func TestJSONQueryRefusesAStringOrNullNeedle(t *testing.T) { + f := Field[map[string]string]{name: "labels"} + s, n := `{"team":"secret-value"}`, json.Number("1") + var nilPtr *numbers + var nilMap map[string]string + var nilSlice []string + b := []byte(`{"team":"secret-value"}`) + for name, c := range map[string]struct { + needle any + want string + }{ + "*string": {&s, "one JSON string"}, + "*json.Number": {&n, "one JSON string"}, + "nil": {nil, "nil needle"}, + "a nil pointer": {nilPtr, "writes JSON null"}, + "a nil map": {nilMap, "writes JSON null"}, + "a nil slice": {nilSlice, "writes JSON null"}, + "null JSON text": {json.RawMessage(`null`), "writes JSON null"}, + "*[]byte": {&b, "base64"}, + } { + for _, operator := range []string{"contains", "contained_by"} { + _, err := JSONQuery[[]byte](context.Background(), f, nil, operator, c.needle) + if !errors.Is(err, encrypt.ErrEncoding) || !strings.Contains(err.Error(), c.want) || strings.Contains(err.Error(), "secret-value") { + t.Errorf("%s %s: %v, want %q", operator, name, err, c.want) + } + } + } + // A needle that writes itself, a pointer to one included, and an empty + // map pass the checks and reach the cipher, which this test has none of. + raw := json.RawMessage(`{"team":"red"}`) + for name, needle := range map[string]any{"*json.RawMessage": &raw, "an empty map": map[string]string{}, "a map": map[string]string{"team": "red"}} { + if _, err := JSONQuery[[]byte](context.Background(), f, nil, "contains", needle); err == nil || !strings.Contains(err.Error(), "needs a cipher") { + t.Errorf("%s: %v, want it to reach the cipher", name, err) + } + } +} diff --git a/languages/golang/encrypt/internal/testusers/customer_stash.go b/languages/golang/encrypt/internal/testusers/customer_stash.go index d2d8347c5..5a822ad37 100644 --- a/languages/golang/encrypt/internal/testusers/customer_stash.go +++ b/languages/golang/encrypt/internal/testusers/customer_stash.go @@ -141,19 +141,21 @@ func (f CustomerProfileField) Encrypt(ctx context.Context, c *encrypt.Cipher, v // Contains derives the query for documents that contain needle: col @> query. // needle is a json.RawMessage of JSON text, or a map, struct or slice that -// encoding/json writes; a Go string is refused, as one JSON string is not JSON -// text. Each member it writes must be in the document, so a zero-valued struct -// field narrows the query to documents holding that zero: give a needle struct -// pointer fields, or omitempty, or use a map. Each array element of the needle -// must sit inside one element of the document's array. +// encoding/json writes; a Go string or a pointer to one is refused, as one JSON +// string is not JSON text, and so is a needle that writes JSON null (nil, or a +// nil pointer, map or slice). Each member it writes must be in the document, so +// a zero-valued struct field narrows the query to documents holding that zero: +// give a needle struct pointer fields, or omitempty, or use a map. Each array +// element of the needle must sit inside one element of the document's array. func (f CustomerProfileField) Contains(ctx context.Context, c *encrypt.Cipher, needle any) (eql.JSONSearchQuery, error) { return gensupport.JSONQuery[eql.JSONSearchQuery](ctx, f.field, c, "contains", needle) } // ContainedBy derives the query for documents contained by needle: col <@ // query. needle is a json.RawMessage of JSON text, or a map, struct or slice -// that encoding/json writes (not a Go string); here a zero-valued field widens -// the query, admitting documents that hold that zero. +// that encoding/json writes (not a Go string or a pointer to one, nor a needle +// that writes JSON null); here a zero-valued field widens the query, admitting +// documents that hold that zero. func (f CustomerProfileField) ContainedBy(ctx context.Context, c *encrypt.Cipher, needle any) (eql.JSONSearchQuery, error) { return gensupport.JSONQuery[eql.JSONSearchQuery](ctx, f.field, c, "contained_by", needle) } @@ -254,19 +256,21 @@ func (f CustomerPrefsField) Encrypt(ctx context.Context, c *encrypt.Cipher, v js // Contains derives the query for documents that contain needle: col @> query. // needle is a json.RawMessage of JSON text, or a map, struct or slice that -// encoding/json writes; a Go string is refused, as one JSON string is not JSON -// text. Each member it writes must be in the document, so a zero-valued struct -// field narrows the query to documents holding that zero: give a needle struct -// pointer fields, or omitempty, or use a map. Each array element of the needle -// must sit inside one element of the document's array. +// encoding/json writes; a Go string or a pointer to one is refused, as one JSON +// string is not JSON text, and so is a needle that writes JSON null (nil, or a +// nil pointer, map or slice). Each member it writes must be in the document, so +// a zero-valued struct field narrows the query to documents holding that zero: +// give a needle struct pointer fields, or omitempty, or use a map. Each array +// element of the needle must sit inside one element of the document's array. func (f CustomerPrefsField) Contains(ctx context.Context, c *encrypt.Cipher, needle any) (eql.JSONSearchQuery, error) { return gensupport.JSONQuery[eql.JSONSearchQuery](ctx, f.field, c, "contains", needle) } // ContainedBy derives the query for documents contained by needle: col <@ // query. needle is a json.RawMessage of JSON text, or a map, struct or slice -// that encoding/json writes (not a Go string); here a zero-valued field widens -// the query, admitting documents that hold that zero. +// that encoding/json writes (not a Go string or a pointer to one, nor a needle +// that writes JSON null); here a zero-valued field widens the query, admitting +// documents that hold that zero. func (f CustomerPrefsField) ContainedBy(ctx context.Context, c *encrypt.Cipher, needle any) (eql.JSONSearchQuery, error) { return gensupport.JSONQuery[eql.JSONSearchQuery](ctx, f.field, c, "contained_by", needle) } diff --git a/languages/golang/encrypt/internal/testusers/subscriber_stash.go b/languages/golang/encrypt/internal/testusers/subscriber_stash.go index 3e7f7a3f0..d0174c9ba 100644 --- a/languages/golang/encrypt/internal/testusers/subscriber_stash.go +++ b/languages/golang/encrypt/internal/testusers/subscriber_stash.go @@ -151,20 +151,24 @@ func (f SubscriberProfileField) Encrypt(ctx context.Context, c *encrypt.Cipher, // Contains derives the query for documents that contain needle: col @> query. // needle is a json.RawMessage of JSON text, or a map, struct or slice that -// encoding/json writes; a Go string is refused, as one JSON string is not JSON -// text. Each member it writes must be in the document, so a zero-valued struct -// field narrows the query to documents holding that zero: give a needle struct -// pointer fields, or omitempty, or use a map. Each array element of the needle -// must sit inside one element of the document's array. A needle of the field's -// own type, SubscriberProfile, is refused for that reason. +// encoding/json writes; a Go string or a pointer to one is refused, as one JSON +// string is not JSON text, and so is a needle that writes JSON null (nil, or a +// nil pointer, map or slice). Each member it writes must be in the document, so +// a zero-valued struct field narrows the query to documents holding that zero: +// give a needle struct pointer fields, or omitempty, or use a map. Each array +// element of the needle must sit inside one element of the document's array. A +// needle of the field's own type, SubscriberProfile, is refused for that +// reason, and so is a pointer to it: write a needle type whose fields are +// pointers with omitempty, or use a map. func (f SubscriberProfileField) Contains(ctx context.Context, c *encrypt.Cipher, needle any) (eql.JSONSearchQuery, error) { return gensupport.JSONQuery[eql.JSONSearchQuery](ctx, f.field, c, "contains", needle) } // ContainedBy derives the query for documents contained by needle: col <@ // query. needle is a json.RawMessage of JSON text, or a map, struct or slice -// that encoding/json writes (not a Go string); here a zero-valued field widens -// the query, admitting documents that hold that zero. +// that encoding/json writes (not a Go string or a pointer to one, nor a needle +// that writes JSON null); here a zero-valued field widens the query, admitting +// documents that hold that zero. func (f SubscriberProfileField) ContainedBy(ctx context.Context, c *encrypt.Cipher, needle any) (eql.JSONSearchQuery, error) { return gensupport.JSONQuery[eql.JSONSearchQuery](ctx, f.field, c, "contained_by", needle) } @@ -265,20 +269,24 @@ func (f SubscriberPrefsField) Encrypt(ctx context.Context, c *encrypt.Cipher, v // Contains derives the query for documents that contain needle: col @> query. // needle is a json.RawMessage of JSON text, or a map, struct or slice that -// encoding/json writes; a Go string is refused, as one JSON string is not JSON -// text. Each member it writes must be in the document, so a zero-valued struct -// field narrows the query to documents holding that zero: give a needle struct -// pointer fields, or omitempty, or use a map. Each array element of the needle -// must sit inside one element of the document's array. A needle of the field's -// own type, SubscriberPrefs, is refused for that reason. +// encoding/json writes; a Go string or a pointer to one is refused, as one JSON +// string is not JSON text, and so is a needle that writes JSON null (nil, or a +// nil pointer, map or slice). Each member it writes must be in the document, so +// a zero-valued struct field narrows the query to documents holding that zero: +// give a needle struct pointer fields, or omitempty, or use a map. Each array +// element of the needle must sit inside one element of the document's array. A +// needle of the field's own type, SubscriberPrefs, is refused for that reason, +// and so is a pointer to it: write a needle type whose fields are pointers with +// omitempty, or use a map. func (f SubscriberPrefsField) Contains(ctx context.Context, c *encrypt.Cipher, needle any) (eql.JSONSearchQuery, error) { return gensupport.JSONQuery[eql.JSONSearchQuery](ctx, f.field, c, "contains", needle) } // ContainedBy derives the query for documents contained by needle: col <@ // query. needle is a json.RawMessage of JSON text, or a map, struct or slice -// that encoding/json writes (not a Go string); here a zero-valued field widens -// the query, admitting documents that hold that zero. +// that encoding/json writes (not a Go string or a pointer to one, nor a needle +// that writes JSON null); here a zero-valued field widens the query, admitting +// documents that hold that zero. func (f SubscriberPrefsField) ContainedBy(ctx context.Context, c *encrypt.Cipher, needle any) (eql.JSONSearchQuery, error) { return gensupport.JSONQuery[eql.JSONSearchQuery](ctx, f.field, c, "contained_by", needle) } @@ -337,19 +345,21 @@ func (f SubscriberLabelsField) Encrypt(ctx context.Context, c *encrypt.Cipher, v // Contains derives the query for documents that contain needle: col @> query. // needle is a json.RawMessage of JSON text, or a map, struct or slice that -// encoding/json writes; a Go string is refused, as one JSON string is not JSON -// text. Each member it writes must be in the document, so a zero-valued struct -// field narrows the query to documents holding that zero: give a needle struct -// pointer fields, or omitempty, or use a map. Each array element of the needle -// must sit inside one element of the document's array. +// encoding/json writes; a Go string or a pointer to one is refused, as one JSON +// string is not JSON text, and so is a needle that writes JSON null (nil, or a +// nil pointer, map or slice). Each member it writes must be in the document, so +// a zero-valued struct field narrows the query to documents holding that zero: +// give a needle struct pointer fields, or omitempty, or use a map. Each array +// element of the needle must sit inside one element of the document's array. func (f SubscriberLabelsField) Contains(ctx context.Context, c *encrypt.Cipher, needle any) (eql.JSONSearchQuery, error) { return gensupport.JSONQuery[eql.JSONSearchQuery](ctx, f.field, c, "contains", needle) } // ContainedBy derives the query for documents contained by needle: col <@ // query. needle is a json.RawMessage of JSON text, or a map, struct or slice -// that encoding/json writes (not a Go string); here a zero-valued field widens -// the query, admitting documents that hold that zero. +// that encoding/json writes (not a Go string or a pointer to one, nor a needle +// that writes JSON null); here a zero-valued field widens the query, admitting +// documents that hold that zero. func (f SubscriberLabelsField) ContainedBy(ctx context.Context, c *encrypt.Cipher, needle any) (eql.JSONSearchQuery, error) { return gensupport.JSONQuery[eql.JSONSearchQuery](ctx, f.field, c, "contained_by", needle) } diff --git a/languages/golang/stashgen/emit.go b/languages/golang/stashgen/emit.go index c25fe9b29..5c7ec4fe9 100644 --- a/languages/golang/stashgen/emit.go +++ b/languages/golang/stashgen/emit.go @@ -1123,9 +1123,12 @@ func (w *writer) jsonEntry(g genField) { recv := "func (f " + g.fieldType + ") " if answered["contains"] { w.nl() - doc := "Contains derives the query for documents that contain needle: col @> query. needle is a json.RawMessage of JSON text, or a map, struct or slice that encoding/json writes; a Go string is refused, as one JSON string is not JSON text. Each member it writes must be in the document, so a zero-valued struct field narrows the query to documents holding that zero: give a needle struct pointer fields, or omitempty, or use a map. Each array element of the needle must sit inside one element of the document's array." - if g.holdsStruct { - doc += " A needle of the field's own type, " + g.valueExpr + ", is refused for that reason." + doc := "Contains derives the query for documents that contain needle: col @> query. needle is a json.RawMessage of JSON text, or a map, struct or slice that encoding/json writes; a Go string or a pointer to one is refused, as one JSON string is not JSON text, and so is a needle that writes JSON null (nil, or a nil pointer, map or slice). Each member it writes must be in the document, so a zero-valued struct field narrows the query to documents holding that zero: give a needle struct pointer fields, or omitempty, or use a map. Each array element of the needle must sit inside one element of the document's array." + switch { + case g.structElems: + doc += " A needle of the field's own type, " + g.valueExpr + ", is refused for that reason, and so is a pointer to it: each struct it holds writes its zero fields too. Write a needle type whose fields are pointers with omitempty, and pass a slice or map of it in place of " + g.valueExpr + " (a []VisitNeedle for a []Visit)." + case g.holdsStruct: + doc += " A needle of the field's own type, " + g.valueExpr + ", is refused for that reason, and so is a pointer to it: write a needle type whose fields are pointers with omitempty, or use a map." } w.b.WriteString(wrapComment(doc, 80)) w.p("%sContains(ctx context.Context, c *encrypt.Cipher, needle any) (%s, error) {", recv, needle) @@ -1134,7 +1137,7 @@ func (w *writer) jsonEntry(g genField) { } if answered["contained_by"] { w.nl() - w.b.WriteString(wrapComment("ContainedBy derives the query for documents contained by needle: col <@ query. needle is a json.RawMessage of JSON text, or a map, struct or slice that encoding/json writes (not a Go string); here a zero-valued field widens the query, admitting documents that hold that zero.", 80)) + w.b.WriteString(wrapComment("ContainedBy derives the query for documents contained by needle: col <@ query. needle is a json.RawMessage of JSON text, or a map, struct or slice that encoding/json writes (not a Go string or a pointer to one, nor a needle that writes JSON null); here a zero-valued field widens the query, admitting documents that hold that zero.", 80)) w.p("%sContainedBy(ctx context.Context, c *encrypt.Cipher, needle any) (%s, error) {", recv, needle) w.p("\treturn gensupport.JSONQuery[%s](ctx, f.field, c, \"contained_by\", needle)", needle) w.p("}") diff --git a/languages/golang/stashgen/read.go b/languages/golang/stashgen/read.go index 7ad8cd2f5..a67898e34 100644 --- a/languages/golang/stashgen/read.go +++ b/languages/golang/stashgen/read.go @@ -111,6 +111,9 @@ type genField struct { // holdsStruct is set for a document field (KindJSONValue) whose type // holds a struct: its Contains refuses a needle of that type. holdsStruct bool + // structElems is set with holdsStruct when the type is a slice, array + // or map whose elements hold the struct, for the Contains comment. + structElems bool } // definedScalar reports whether the field's type is a named type over a @@ -724,7 +727,7 @@ func (r *reader) buildFields(c *collected) error { // writes and reads: a struct, map or slice, or a pointer to one. // What encoding/json cannot carry both ways is refused here, when // the code is generated. - holdsStruct := false + holdsStruct, structElems := false, false if eqlType := r.eqlType(field.EQLType); field.Verb == VerbEncryptInto && eqlType != nil && eqlType.wire == record.JSON { doc, docType := &field.GoType, cf.typ if doc.Kind == KindPointer && doc.Elem != nil { @@ -735,6 +738,8 @@ func (r *reader) buildFields(c *collected) error { if reason, holdsStruct = notDocument(docType); reason != "" { return fieldErr(typeName, cf.goName, "%s seals a json document, and %s %s; %s", field.EQLType, r.typeExpr(docType), reason, documentTypes) } + _, isStruct := docType.Underlying().(*types.Struct) + structElems = holdsStruct && !isStruct *doc = GoType{Name: doc.Name, Kind: KindJSONValue} } } @@ -772,7 +777,7 @@ func (r *reader) buildFields(c *collected) error { if t.Omit { continue } - g := genField{Field: field, typeExpr: r.typeExpr(cf.typ), via: cf.via, tags: cf.tags, pathType: pathType(cf.typ), holdsStruct: holdsStruct} + g := genField{Field: field, typeExpr: r.typeExpr(cf.typ), via: cf.via, tags: cf.tags, pathType: pathType(cf.typ), holdsStruct: holdsStruct, structElems: structElems} g.valueExpr = g.typeExpr if pointer { g.valueExpr = field.GoType.Name From ed851e2bdd4690d7a1422809cca4355a237746a6 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 21:20:43 +1100 Subject: [PATCH 10/12] test(go): give TestRunGeneratesTypedJSONDocuments its own doc comment The test went in between TestRunGeneratesJSONSearchFields and that test's doc comment, so the comment described the wrong function. It is back above TestRunGeneratesJSONSearchFields. Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- languages/golang/cmd/stashgen/main_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/languages/golang/cmd/stashgen/main_test.go b/languages/golang/cmd/stashgen/main_test.go index 41cc29cd9..4bcf2cf9d 100644 --- a/languages/golang/cmd/stashgen/main_test.go +++ b/languages/golang/cmd/stashgen/main_test.go @@ -291,10 +291,6 @@ func TestModelFlagsRepeat(t *testing.T) { } } -// A json.RawMessage field stored as an encrypted JSON document, against the -// embedded engine with the EQL types: the generated entry has the -// document's methods and an accessor per ordered path, and every refusal -// names the field before anything is written. // A document field of a Go type: encoding/json writes it when it is sealed // and reads it back when it is opened, and the generator refuses, when the // code is generated, a type that it cannot carry both ways. @@ -399,6 +395,10 @@ func TestRunGeneratesTypedJSONDocuments(t *testing.T) { } } +// A json.RawMessage field stored as an encrypted JSON document, against the +// embedded engine with the EQL types: the generated entry has the +// document's methods and an accessor per ordered path, and every refusal +// names the field before anything is written. func TestRunGeneratesJSONSearchFields(t *testing.T) { checker, err := encrypt.NewChecker(context.Background()) if err != nil { From 8591f2785ac46398060c8d48ccc32b1a448ac3bd Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 22:02:05 +1100 Subject: [PATCH 11/12] test(go): expect the typed documents' refusal of a type defined over RawMessage The base's test for a type defined over json.RawMessage expected the refusal the RawMessage-only generator gives. Here a document may be any type encoding/json writes, and the refusal is the one for a byte slice written as base64; the test and the stashgen README say that reason. Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- languages/golang/cmd/stashgen/README.md | 2 +- languages/golang/cmd/stashgen/main_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/languages/golang/cmd/stashgen/README.md b/languages/golang/cmd/stashgen/README.md index ed469e19e..60e6f0aeb 100644 --- a/languages/golang/cmd/stashgen/README.md +++ b/languages/golang/cmd/stashgen/README.md @@ -161,7 +161,7 @@ A `json.RawMessage` comes back as JSON text that holds the same values; key orde A pointer field (`*Profile`, `*json.RawMessage`) is optional and must say `null=sql`: a document has no fixed length to hide a missing one behind. A nil pointer is SQL `NULL`; a non-nil `*json.RawMessage` holding `null` is a stored JSON `null`. A `json.RawMessage` sealed any other way (`encrypt`, `index=equality`) is bytes, as any type defined over `[]byte` is. -`JsonSearch` takes `json.RawMessage` itself, or an alias of it (`type Doc = json.RawMessage`), and refuses a type defined over it (`type Doc json.RawMessage`): `stashgen` knows a document by the `json.RawMessage` type, where a scalar type is accepted with any type defined over it. +`JsonSearch` takes `json.RawMessage` itself, or an alias of it (`type Doc = json.RawMessage`), and refuses a type defined over it (`type Doc json.RawMessage`): such a type has no `MarshalJSON`, so `encoding/json` would write it as a base64 string, not as the JSON text it holds, where a scalar type is accepted with any type defined over it. The type takes options in parentheses: `ordered=` names the paths whose numbers and strings carry an ordering term, separated by `;`, and `sealed_root=false` turns off the sealed root (on by default). `encrypt_into=JsonSearch(ordered=$.age;$.visits[*].at)` orders two paths. diff --git a/languages/golang/cmd/stashgen/main_test.go b/languages/golang/cmd/stashgen/main_test.go index a78a4e035..3aa884c5d 100644 --- a/languages/golang/cmd/stashgen/main_test.go +++ b/languages/golang/cmd/stashgen/main_test.go @@ -483,7 +483,7 @@ func TestRunGeneratesJSONSearchFields(t *testing.T) { {"two paths, one accessor", "\tProfile json.RawMessage `stash:\"profile,encrypt_into=JsonSearch(ordered=$.a_b;$.aB)\"`", "both name the accessor At.AB"}, {"the same ordered path twice", "\tProfile json.RawMessage `stash:\"profile,encrypt_into=JsonSearch(ordered=$.a;$.a)\"`", "the ordered path $.a is given twice"}, {"a path with no accessor name", "\tProfile json.RawMessage `stash:\"profile,encrypt_into=JsonSearch(ordered=$.__)\"`", "no Go accessor name"}, - {"a type defined over json.RawMessage", "\tProfile Doc `stash:\"profile,encrypt_into=JsonSearch\"`", "Customer.Profile: JsonSearch seals a json document: declare the field json.RawMessage"}, + {"a type defined over json.RawMessage", "\tProfile Doc `stash:\"profile,encrypt_into=JsonSearch\"`", "Customer.Profile: JsonSearch seals a json document, and Doc is written by encoding/json as a base64 string"}, {"a string into a document", "\tProfile string `stash:\"profile,encrypt_into=JsonSearch\"`", "JsonSearch seals a json"}, {"options on a type that takes none", "\tEmail string `stash:\"email,encrypt_into=TextEq(ordered=$.a)\"`", "the engine refuses the declaration"}, } { From 5227d0f91df5e97c79d0c17c0415e129d0d14f73 Mon Sep 17 00:00:00 2001 From: Dan Draper Date: Mon, 26 Oct 2026 22:02:05 +1100 Subject: [PATCH 12/12] test(go): compare a typed needle's query with its JSON text's as sets A containment query now sends its selector set in a random order of its own (#1379), so two derivations of one document, typed or as text, serialize differently. The test that holds a typed needle to the query its JSON text derives compared bytes; it now sorts every list of both queries, recursively, and compares those. Claude-Session: https://claude.ai/code/session_01N7bCqa3ZVJMcEE99RzVqEu --- languages/golang/encrypt/json_typed_test.go | 67 +++++++++++++++++++-- 1 file changed, 63 insertions(+), 4 deletions(-) diff --git a/languages/golang/encrypt/json_typed_test.go b/languages/golang/encrypt/json_typed_test.go index 3e5647f41..af9501fbf 100644 --- a/languages/golang/encrypt/json_typed_test.go +++ b/languages/golang/encrypt/json_typed_test.go @@ -1,12 +1,12 @@ package encrypt_test import ( - "bytes" "context" "encoding/json" "errors" "math" "reflect" + "sort" "strings" "testing" @@ -83,8 +83,9 @@ func TestASubscribersTypedNeedlesDeriveTheRawQuery(t *testing.T) { cipher := c.DefaultKeyset() f := testusers.SubscriberFields.Profile // same derives a query twice, once from a Go value and once from the - // JSON text encoding/json writes for it, and wants the same bytes: a - // typed needle is that text, nothing more. + // JSON text encoding/json writes for it, and wants the same selector + // set: a typed needle is that text, nothing more. Each derivation sends + // its set in a random order of its own, so the two compare as sets. same := func(name string, typed, raw func() ([]byte, error)) { t.Helper() a, err := typed() @@ -95,7 +96,7 @@ func TestASubscribersTypedNeedlesDeriveTheRawQuery(t *testing.T) { if err != nil { t.Fatalf("%s raw: %v", name, err) } - if !bytes.Equal(a, b) { + if !sameSelectorSet(t, a, b) { t.Errorf("%s: the typed needle derives another query than its JSON text", name) } } @@ -189,3 +190,61 @@ func TestASubscribersTypedNeedlesDeriveTheRawQuery(t *testing.T) { } } } + +// sameSelectorSet reports whether two containment queries hold the same +// selector set: the same {s, vs} pairs in the same element groups, and the +// same alternative, whatever order their lists are in (the engine shuffles +// each query's set, as it does a stored document's). +func sameSelectorSet(t *testing.T, a, b []byte) bool { + t.Helper() + canonical := func(text []byte) string { + var query any + if err := json.Unmarshal(text, &query); err != nil { + t.Fatalf("a query is JSON: %v", err) + } + out, err := json.Marshal(sortedSet(query)) + if err != nil { + t.Fatal(err) + } + return string(out) + } + return canonical(a) == canonical(b) +} + +// sortedSet is a query's JSON with every list sorted by its members' +// text, recursively: one spelling for one set. +func sortedSet(v any) any { + switch v := v.(type) { + case map[string]any: + out := make(map[string]any, len(v)) + for k, member := range v { + out[k] = sortedSet(member) + } + return out + case []any: + out := make([]any, len(v)) + keys := make([]string, len(v)) + for i, member := range v { + out[i] = sortedSet(member) + text, _ := json.Marshal(out[i]) + keys[i] = string(text) + } + sort.Sort(byKey{out, keys}) + return out + default: + return v + } +} + +// byKey sorts a list by a key per member. +type byKey struct { + items []any + keys []string +} + +func (b byKey) Len() int { return len(b.items) } +func (b byKey) Less(i, j int) bool { return b.keys[i] < b.keys[j] } +func (b byKey) Swap(i, j int) { + b.items[i], b.items[j] = b.items[j], b.items[i] + b.keys[i], b.keys[j] = b.keys[j], b.keys[i] +}