diff --git a/.claude/commands/mendix/explore.md b/.claude/commands/mendix/explore.md index 3b798999c6..eb7c9c39d1 100644 --- a/.claude/commands/mendix/explore.md +++ b/.claude/commands/mendix/explore.md @@ -109,7 +109,7 @@ LIST IMPACT OF Module.Customer; -- Get context for understanding code DESCRIBE CONTEXT OF Module.MyMicroflow DEPTH 2; --- Full-text search across strings and source +-- Full-text search across strings, element documentation and source SEARCH 'validation'; SEARCH 'Customer'; ``` @@ -128,6 +128,10 @@ mxcli search -p app.mpr "error" -q --format names | head -1 | awk '{print $2}' | # JSON format with jq mxcli search -p app.mpr "Customer" -q --format json | jq -r '.[].qualifiedName' + +# A hit's stringContext says which field matched ("documentation" for the +# documentation of an entity, attribute, constant, page, ...) +mxcli search -p app.mpr "legacy" -q --format json | jq -r '.[] | select(.stringContext == "documentation") | .qualifiedName' ``` ## Available Catalog Tables diff --git a/.claude/skills/fix-issue/findings/mdl-other/2026-10-13-search-misses-words-only-in-documentation-or-log-message.json b/.claude/skills/fix-issue/findings/mdl-other/2026-10-13-search-misses-words-only-in-documentation-or-log-message.json new file mode 100644 index 0000000000..fa106bce68 --- /dev/null +++ b/.claude/skills/fix-issue/findings/mdl-other/2026-10-13-search-misses-words-only-in-documentation-or-log-message.json @@ -0,0 +1 @@ +{"area": "mdl/catalog", "date": "2026-10-13", "symptom": "`mxcli search` / `SEARCH` answers \"No matches found.\" for a word that appears only in an element's documentation (a constant, entity, attribute, page, ...) or only in a microflow's log message, although `select ... from catalog.constants where Description like '%word%'` finds it", "cause": "The strings index was filled by a walk keyed on `Texts$Text` plus a few typed extractions. Documentation and a log message's `StringTemplate.Text` are plain strings, not `Texts$Text`, so the walk could not see them, and the typed extractions listed only microflow and workflow documentation and the log node name", "file": "`mdl/catalog/builder_strings.go` (`buildDocumentationStrings`, `extractLogNodeNames`), `mdl/catalog/tables.go` (`CatalogSchemaVersion` 24 -> 25)", "insight": "A search index fed by walking one stored type misses every field of another type, and nothing reports the gap: the missing field just answers \"No matches found.\". Compare the index against the catalog tables that already hold the field (`objects.Description`, `attributes_data`, ...) instead of walking the model again. Two traps: the `MODULE` row of the `objects` view has `'' as ModuleName`, so a copy keeps no module and a filter on `ModuleName` (excluding Marketplace modules) never matches the module's own documentation - use `CASE WHEN ObjectType = 'MODULE' THEN Name ELSE ModuleName END`; and a typed extraction that already indexes microflow/workflow documentation must be excluded from the generic pass or every hit shows twice. Cached catalogs keep answering \"No matches found.\" until the schema version is bumped. Prove the fix with the pair reverted-vs-restored: both tests fail with the documentation pass and the log_message insert removed", "refs": ["mendixlabs/mxcli#1477"]} diff --git a/.claude/skills/mendix/write-lint-rules/catalog-tables.md b/.claude/skills/mendix/write-lint-rules/catalog-tables.md index a9ddbecced..15f0439007 100644 --- a/.claude/skills/mendix/write-lint-rules/catalog-tables.md +++ b/.claude/skills/mendix/write-lint-rules/catalog-tables.md @@ -81,7 +81,7 @@ Returned by `strings(language = None)`. | `qualified_name` | string | The document the text belongs to: `"Sales.Order_Overview"` | | `object_type` | string | Catalog object type, upper-case: `"PAGE"`, `"SNIPPET"`, `"LAYOUT"`, `"MICROFLOW"`, `"NANOFLOW"`, `"WORKFLOW"`, `"ENUMERATION"`, `"MENU_DOCUMENT"`, … | | `value` | string | The text itself | -| `context` | string | What the text is. Translatable text names its stored type and property: `"Forms$Page.Title"`, `"Forms$TabPage.Caption"`, `"Enumerations$EnumerationValue.Caption"`; the rest a lower-case label: `"documentation"`, `"page_url"`, `"log_node"`, … | +| `context` | string | What the text is. Translatable text names its stored type and property: `"Forms$Page.Title"`, `"Forms$TabPage.Caption"`, `"Enumerations$EnumerationValue.Caption"`; the rest a lower-case label: `"documentation"` (of an entity, attribute, constant, page, … — `object_type` says which; a module's domain model documentation is `"domain_model_documentation"`), `"page_url"`, `"log_node"`, `"log_message"`, … | | `language` | string | `"en_US"`; `""` for text that is not translatable (documentation, URLs) | | `element_id` | string | UUID of the element carrying the text | | `module_name` | string | `"Sales"` | diff --git a/CHANGELOG.md b/CHANGELOG.md index ab8083df83..68cb984941 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ## [Unreleased] +### Fixed + +- **`mxcli search` and `SEARCH` find words that appear only in an element's documentation** (mendixlabs/mxcli#1477) — the strings index held captions, messages, URLs and the documentation of microflows and workflows, but not the documentation of entities, attributes, associations, enumerations, constants, pages and the other documents whose catalog tables carry it, so a word present only there answered "No matches found." (measured: a word in a constant's documentation was found by `select … from catalog.constants where Description like …` and not by `search`). The index now also holds those: each hit names the element (`ObjectType` `ENTITY`, `ATTRIBUTE`, `CONSTANT`, `PAGE`, …; an attribute is `Module.Entity.Attribute`) and the field (`StringContext` `documentation`), the labels microflow documentation already used, plus the parameters of microflows and Java actions and a module's domain model documentation (`domain_model_documentation`). A module's own rows carry the module's name in `ModuleName`, not the empty string the `objects` view holds for a module. They are read from the tables the catalog builder has just filled, not from a second walk of the model; building a catalog of a stock 11.12 app takes the same time (0.5s) and the strings table grows by about 50 rows. Enumeration values are not indexed (Mendix has no documentation field on them), nor is the summary text of a remote service's contract entities. The catalog schema version is bumped to 25, so a cached catalog is rebuilt once. Rules: docs-site "Full-Text Search". +- **`search` finds a microflow's log message** (mendixlabs/mxcli#1477) — only the log node name was indexed. A log message is a plain-text `Microflows$StringTemplate`, not a `Texts$Text`, so the walk that finds every translatable text never saw it, and a word present only in a log message was not found. It is now a `log_message` row beside the `log_node` one (activities inside a loop are not reached, as before for the node name). + ## [0.26.0] - 2026-10-11 Headline: **Documents mxcli writes now survive Studio Pro's merge and diff: every element carries exactly the property names Studio Pro writes for the project's Mendix version.** Before, a key mxcli left out, or wrote for a newer version, made a document unmergeable the first time Studio Pro saved it, although `mx check`, the build and the runtime stayed green. `mxcli fix property-sets` repairs projects an older mxcli wrote. Alongside it: a Claude Code companion mod with a Project pane, `describe` → `exec` round trips that keep loop back-edges, shared error handlers and page design properties, `check -p` resolving script sets and security references the way `exec` does, `mxcli run --local` as a background service, and a run of Windows fixes for the local dev loop. The [language poster](https://www.mxcli.org/poster/mxcli-language-poster.html) puts all of mxcli and MDL on one page. diff --git a/cmd/mxcli/cmd_query.go b/cmd/mxcli/cmd_query.go index 20c05ca1e9..671e9abb7e 100644 --- a/cmd/mxcli/cmd_query.go +++ b/cmd/mxcli/cmd_query.go @@ -227,8 +227,25 @@ var searchCmd = &cobra.Command{ Short: "Search the project catalog using full-text search", Long: `Search the project catalog for matching strings and source code. -Searches across string literals (captions, labels, messages) and MDL source -definitions. Requires at least a FULL catalog build (done automatically). +Searches across string literals (captions, labels, messages), the documentation +of project elements, and MDL source definitions. Requires at least a FULL +catalog build (done automatically). + +What is searched, besides captions, labels and messages, is the documentation of: + entities; attributes; associations; enumerations; constants; pages, snippets, + layouts, building blocks and page templates; microflows, nanoflows, rules and + workflows; Java and JavaScript actions; microflow and Java action parameters; + consumed and published REST services and business event services; import and + export mappings, JSON structures and data transformers; scheduled events, task + queues, regular expressions, image and icon collections and menus; agents, AI + models, knowledge bases and consumed MCP services; and a module's domain model + documentation. That is every document type whose catalog table carries a + documentation text. + Not searched: documentation of an enumeration value (Mendix has no such + field) and the summary text of a remote service's contract. +A hit names the element (qualified name, type) and the field: the context +column says "documentation" for a documentation hit. + MDL source is searched only once the source index exists — build it with mxcli -p app.mpr -c "refresh catalog full source" Until then a warning on stderr says only string literals were searched. diff --git a/cmd/mxcli/syntax/features_misc.go b/cmd/mxcli/syntax/features_misc.go index 302e255647..781d4eb287 100644 --- a/cmd/mxcli/syntax/features_misc.go +++ b/cmd/mxcli/syntax/features_misc.go @@ -1164,7 +1164,7 @@ mxcli -p app.mpr --json -c "LIST FOLDERS IN MyModule" Register(SyntaxFeature{ Path: "search", - Summary: "Full-text search across project strings and source definitions", + Summary: "Full-text search across project strings, element documentation and source definitions", Keywords: []string{ "search", "full-text search", "find", "grep", "fts", "catalog strings", "catalog source", diff --git a/docs-site/src/internals/catalog-schema.md b/docs-site/src/internals/catalog-schema.md index 587000f818..54d708dff6 100644 --- a/docs-site/src/internals/catalog-schema.md +++ b/docs-site/src/internals/catalog-schema.md @@ -368,7 +368,8 @@ CREATE VIRTUAL TABLE STRINGS USING fts5( StringContext, -- Where it lives. For translatable text this is -- ., e.g. Forms$ActionButton.Caption. -- Non-translatable strings keep a plain label: page_url, - -- log_node, documentation, rest_path, task_name, ... + -- log_node, log_message, documentation, rest_path, + -- task_name, domain_model_documentation, ... Language, -- Language code, empty for non-translatable strings ElementId, -- The owning element's $ID — what distinguishes an -- enumeration's twelve values from each other @@ -383,6 +384,14 @@ templates (`PAGE_TEMPLATE`, `BUILDING_BLOCK`), which are roughly 70% of a stock project's text and never render in a running app — filter them out with `ObjectType` when you want only the app's own strings. +The documentation of an element (an entity, attribute, constant, page, ...) is a +plain string, not a `Texts$Text`, so it is added from the catalog tables that +already hold it, not by the walk: `StringContext` is `documentation`, +`ObjectType` names the element (`ENTITY`, `ATTRIBUTE`, `CONSTANT`, `PAGE`, ...) +and `QualifiedName` is the element's own (`Sales.Order.Total` for an +attribute). Microflow and workflow documentation is indexed with the same +labels. See [Full-Text Search](../tools/search.md) for the full list. + An empty translation — a text that exists but is not translated yet — is **not** a row, so a language's presence in this table means it is actually translated somewhere. diff --git a/docs-site/src/reference/query/search.md b/docs-site/src/reference/query/search.md index e6e86425e6..1e380626d5 100644 --- a/docs-site/src/reference/query/search.md +++ b/docs-site/src/reference/query/search.md @@ -6,14 +6,14 @@ ## Description -Performs a full-text search across all strings, messages, captions, and MDL source in the project. Uses FTS5 (SQLite full-text search) for fast matching. Returns results grouped by document type and location. +Performs a full-text search across all strings, messages, captions, the documentation of project elements (entities, attributes, constants, pages and more), and MDL source in the project. Uses FTS5 (SQLite full-text search) for fast matching. Returns results grouped by document type and location. This command requires the catalog to be populated. If search returns no results, run `REFRESH CATALOG FULL` first. The search is also available as a CLI subcommand: `mxcli search -p app.mpr "keyword" --format names|json`. ## Parameters *keyword* -: The search term to look for. Must be enclosed in single quotes. Matches against entity names, attribute names, microflow activity captions, page titles, widget labels, string constants, log messages, and MDL source text. +: The search term to look for. Must be enclosed in single quotes. Matches against entity names, attribute names, microflow activity captions, page titles, widget labels, string constants, log messages, element documentation, and MDL source text. ## Examples diff --git a/docs-site/src/tools/search.md b/docs-site/src/tools/search.md index 81c6a0f06e..47b4efb05b 100644 --- a/docs-site/src/tools/search.md +++ b/docs-site/src/tools/search.md @@ -1,6 +1,6 @@ # Full-Text Search -The `SEARCH` command performs full-text search across all strings and source code in a Mendix project. It searches element names, documentation, captions, expressions, and other string content. +The `SEARCH` command performs full-text search across all strings and source code in a Mendix project. It searches captions, messages, the documentation of project elements (entities, attributes, constants, pages and more), and MDL source. ## Syntax @@ -69,16 +69,38 @@ mxcli search -p app.mpr "error" -q --format names | head -1 | awk '{print $2}' | ## What Gets Searched -The search command looks through: - -- Element names (entities, microflows, pages, etc.) -- Documentation comments -- Attribute names and captions -- Expression strings in microflows -- Widget captions and labels -- Enumeration value captions -- Log message strings -- Validation feedback messages +`SEARCH` looks in two indexes, and says which one a hit came from. + +**Strings** (a full catalog, built automatically): + +- Every translatable text: page titles, widget captions and labels, enumeration value captions, button and menu captions, show-message and validation-feedback texts, in every language +- Log messages and log node names of microflows +- Page URLs and the paths, resources and operation summaries of published REST services +- Workflow names, descriptions, task names and activity captions +- Element names: the qualified name is a searchable column too, so `SEARCH 'Customer'` also returns the rows of elements named Customer that have some indexed text +- **Documentation of project elements.** A hit's `ObjectType` names the element and its `StringContext` is `documentation`: + + | Documented element | `ObjectType` | + |--------------------|--------------| + | Entity | `ENTITY` | + | Attribute (qualified `Module.Entity.Attribute`) | `ATTRIBUTE` | + | Association | `ASSOCIATION` | + | Enumeration | `ENUMERATION` | + | Constant | `CONSTANT` | + | Page, snippet, layout, building block, page template | `PAGE`, `SNIPPET`, `LAYOUT`, `BUILDING_BLOCK`, `PAGE_TEMPLATE` | + | Microflow, nanoflow, rule | `MICROFLOW`, `NANOFLOW`, `RULE` | + | Workflow | `WORKFLOW` | + | Java action, JavaScript action | `JAVA_ACTION`, `JAVASCRIPT_ACTION` | + | Microflow parameter, Java action parameter (qualified `Module.Flow.Parameter`) | `MICROFLOW_PARAMETER`, `JAVA_ACTION_PARAMETER` | + | Consumed and published REST service, business event service | `CONSUMED_REST_SERVICE`, `PUBLISHED_REST_SERVICE`, `BUSINESS_EVENT_SERVICE` | + | Import mapping, export mapping, JSON structure, data transformer | `IMPORT_MAPPING`, `EXPORT_MAPPING`, `JSON_STRUCTURE`, `DATA_TRANSFORMER` | + | Scheduled event, task queue, regular expression, image and icon collection, menu | `SCHEDULED_EVENT`, `TASK_QUEUE`, `REGULAR_EXPRESSION`, `IMAGE_COLLECTION`, `ICON_COLLECTION`, `MENU` | + | Agent, AI model, knowledge base, consumed MCP service | `AGENT`, `AI_MODEL`, `KNOWLEDGE_BASE`, `CONSUMED_MCP_SERVICE` | + | A module's domain model documentation | `MODULE`, with `StringContext` `domain_model_documentation` | + + This is the complete set: every document type whose catalog table carries a documentation text. Not covered: the documentation of an enumeration value (Mendix has no such field) and the summary text of a remote service's contract entities (written by the remote service, not by the project). + +**MDL source** (only after `refresh catalog full source`): microflow, nanoflow and page bodies, expressions and the other definitions as MDL text. Until it is built, a warning on stderr says only strings were searched. ## Comparison with Catalog Queries diff --git a/mdl-examples/bug-tests/1477-search-documentation.mdl b/mdl-examples/bug-tests/1477-search-documentation.mdl new file mode 100644 index 0000000000..0b52b786d3 --- /dev/null +++ b/mdl-examples/bug-tests/1477-search-documentation.mdl @@ -0,0 +1,46 @@ +mdl 1; +-- ============================================================================ +-- mendixlabs/mxcli#1477: SEARCH finds words that appear only in documentation +-- ============================================================================ +-- +-- Before: the strings index held captions, messages and URLs, but not the +-- documentation of a constant, an entity or an attribute, and only the name of +-- a microflow's log node, not its message. A word present only in one of those +-- fields answered "No matches found.", although the catalog table that holds +-- the documentation found it: +-- +-- select QualifiedName from catalog.constants where Description like '%legacy rate%' +-- +-- Documentation and the text of a log message are plain strings, not +-- Texts$Text, so the walk that finds every translatable text never saw them. +-- +-- After: each of the words below is a SEARCH hit. The hit names the element +-- (ObjectType CONSTANT, ENTITY, ATTRIBUTE, MICROFLOW) and the field +-- (StringContext documentation, log_message). +-- +-- Verify (needs a project; `check` only proves the script is valid MDL): +-- +-- mxcli exec 1477-search-documentation.mdl -p app.mpr +-- mxcli search -p app.mpr "legacy rate" -- BugTest1477.RateUrl, documentation +-- mxcli search -p app.mpr "settled invoices" -- BugTest1477.Invoice, documentation +-- mxcli search -p app.mpr "gross amount" -- BugTest1477.Invoice.Amount, documentation +-- mxcli search -p app.mpr "rate lookup" -- BugTest1477.ACT_Lookup, log_message +-- +-- Then perturb: search for a word that is in none of them and see +-- "No matches found.". + +create module BugTest1477; + +/** legacy rate source */ +create constant BugTest1477.RateUrl (Type: String, DefaultValue: 'https://rates.example.com'); + +/** Settled invoices only */ +create persistent entity BugTest1477.Invoice ( + /** The gross amount before discount */ + Amount: Decimal +); + +create microflow BugTest1477.ACT_Lookup () +begin + log info node 'Rates' 'Starting rate lookup'; +end; diff --git a/mdl/catalog/builder_strings.go b/mdl/catalog/builder_strings.go index 5f20eaf9f6..2c1bee5c4f 100644 --- a/mdl/catalog/builder_strings.go +++ b/mdl/catalog/builder_strings.go @@ -68,8 +68,9 @@ func (b *Builder) buildStrings() error { mfID := string(mf.ID) - // Documentation (no language). The activities' message templates - // are Texts$Text and come from the walk. + // Documentation (no language). A log message is a plain-text + // StringTemplate and is indexed with its node name below; the + // other message templates are Texts$Text and come from the walk. if mf.Documentation != "" { insert(qn, "MICROFLOW", mf.Documentation, "documentation", "", mfID, moduleName) } @@ -131,10 +132,82 @@ func (b *Builder) buildStrings() error { b.buildTranslatableStrings(insert) + docRows, err := b.buildDocumentationStrings() + if err != nil { + return err + } + count += docRows + b.report("strings", count) return nil } +// buildDocumentationStrings indexes the documentation of every element whose +// catalog table already holds it: entities, attributes, associations, +// enumerations, constants, pages, snippets, layouts, nanoflows, Java actions, +// REST services, mappings and the rest of the `objects` view, plus the +// parameters of microflows and Java actions and a module's domain model +// documentation. +// +// Documentation is a plain string, not a Texts$Text, so neither the translatable +// walk nor the typed extractions above reach it. It is read back from the tables +// the builders above just filled rather than walked from the model a second +// time. Each row names its element (QualifiedName, ObjectType) and its field +// (StringContext), the same labels the typed microflow and workflow +// documentation rows use. +// +// Left out on purpose: +// - MICROFLOW and WORKFLOW — their documentation is indexed by the typed +// extractions in buildStrings, and a second row would make every hit appear +// twice. +// - CONTRACT_ENTITY — its Description is a remote service's $metadata summary, +// not text the project's author wrote. +// - enumeration values — the metamodel has no documentation property on them. +// +// A module is its own module: the MODULE row of the objects view carries an +// empty ModuleName, so the module's documentation is stored under its Name. +// Without that, a filter on ModuleName (module-scoped searches, excluding +// Marketplace modules) never matches the module's own text. +// +// Rows are limited to the snapshot being built, so a catalog that also holds +// another project's snapshot does not index that project's text under this one. +func (b *Builder) buildDocumentationStrings() (int, error) { + _, snapshotID := b.snapshotMeta() + + res, err := b.tx.Exec(` + INSERT INTO strings (QualifiedName, ObjectType, StringValue, StringContext, Language, ElementId, ModuleName) + SELECT QualifiedName, ObjectType, Description, 'documentation', '', Id, + CASE WHEN ObjectType = 'MODULE' THEN Name ELSE ModuleName END + FROM objects + WHERE Description <> '' AND SnapshotId = ?1 + AND ObjectType NOT IN ('MICROFLOW', 'WORKFLOW', 'CONTRACT_ENTITY') + UNION ALL + SELECT EntityQualifiedName || '.' || Name, 'ATTRIBUTE', Description, 'documentation', '', Id, ModuleName + FROM attributes_data + WHERE Description <> '' AND SnapshotId = ?1 + UNION ALL + SELECT MicroflowQualifiedName || '.' || Name, 'MICROFLOW_PARAMETER', Description, 'documentation', '', Id, ModuleName + FROM microflow_parameters_data + WHERE Description <> '' AND SnapshotId = ?1 + UNION ALL + SELECT QualifiedName, 'JAVA_ACTION_PARAMETER', Description, 'documentation', '', Id, ModuleName + FROM java_action_parameters_data + WHERE Description <> '' AND SnapshotId = ?1 + UNION ALL + SELECT QualifiedName, 'MODULE', DomainModelDocumentation, 'domain_model_documentation', '', Id, Name + FROM modules_data + WHERE DomainModelDocumentation <> '' AND SnapshotId = ?1 + `, snapshotID) + if err != nil { + return 0, err + } + n, err := res.RowsAffected() + if err != nil { + return 0, err + } + return int(n), nil +} + // buildTranslatableStrings indexes every Texts$Text in the project. // // It walks the RAW units rather than the typed readers, deliberately. The typed @@ -306,10 +379,13 @@ func extractWorkflowFlowStrings(flow *workflows.Flow, qn, moduleName string, ins } } -// extractLogNodeNames indexes the one microflow-activity string that is NOT a -// Texts$Text. The message templates that used to be extracted here — log, show -// message, validation feedback — are Texts$Text and come from the walk in -// buildTranslatableStrings, which also reaches the ones this never listed. +// extractLogNodeNames indexes the two microflow-activity strings that are NOT a +// Texts$Text: a log node's name and a log message's text. A log message is a +// Microflows$StringTemplate, whose Text is a plain string, so the translatable +// walk in buildTranslatableStrings never sees it (measured on a Mendix 11.12 +// project: a word present only in a log message was not found by SEARCH). The +// other message templates — show message, validation feedback — are Texts$Text +// and come from the walk, which also reaches the ones this never listed. func extractLogNodeNames(oc *microflows.MicroflowObjectCollection, qn, objType, moduleName string, insert func(string, string, string, string, string, string, string)) { if oc == nil { return @@ -319,8 +395,15 @@ func extractLogNodeNames(oc *microflows.MicroflowObjectCollection, qn, objType, if !ok || act.Action == nil { continue } - if a, ok := act.Action.(*microflows.LogMessageAction); ok && a.LogNodeName != "" { + a, ok := act.Action.(*microflows.LogMessageAction) + if !ok { + continue + } + if a.LogNodeName != "" { insert(qn, objType, a.LogNodeName, "log_node", "", string(act.ID), moduleName) } + if msg := textValue(a.MessageTemplate); msg != "" { + insert(qn, objType, msg, "log_message", "", string(act.ID), moduleName) + } } } diff --git a/mdl/catalog/builder_strings_documentation_test.go b/mdl/catalog/builder_strings_documentation_test.go new file mode 100644 index 0000000000..203100c47b --- /dev/null +++ b/mdl/catalog/builder_strings_documentation_test.go @@ -0,0 +1,241 @@ +// SPDX-License-Identifier: Apache-2.0 + +package catalog + +import ( + "testing" + + "github.com/mendixlabs/mxcli/mdl/backend/mock" + "github.com/mendixlabs/mxcli/model" + "github.com/mendixlabs/mxcli/sdk/domainmodel" + "github.com/mendixlabs/mxcli/sdk/microflows" + "github.com/mendixlabs/mxcli/sdk/pages" +) + +// docFixture builds the pieces of a project that carry documentation, runs the +// builders that fill the catalog tables plus buildStrings in full mode, and +// returns the catalog. One word appears only in documentation fields. +func docFixture(t *testing.T, word string) *Catalog { + t.Helper() + const modID = model.ID("mod-sales") + + entity := &domainmodel.Entity{ + BaseElement: model.BaseElement{ID: "ent-order"}, + Name: "Order", + Documentation: note("Entity notes: ", word), + Persistable: true, + Attributes: []*domainmodel.Attribute{{ + BaseElement: model.BaseElement{ID: "attr-total"}, + Name: "Total", + Documentation: note("Attribute notes: ", word), + Type: &domainmodel.StringAttributeType{}, + }}, + } + entity.ContainerID = "dm-sales" + dm := &domainmodel.DomainModel{ + BaseElement: model.BaseElement{ID: "dm-sales"}, + ContainerID: modID, + Documentation: note("Domain model notes: ", word), + Entities: []*domainmodel.Entity{entity}, + } + constant := &model.Constant{ + BaseElement: model.BaseElement{ID: "const-limit"}, + ContainerID: modID, + Name: "Limit", + Documentation: note("Constant notes: ", word), + Type: model.ConstantDataType{Kind: "String"}, + } + page := &pages.Page{ + BaseElement: model.BaseElement{ID: "page-home"}, + ContainerID: modID, + Name: "Home", + Documentation: note("Page notes: ", word), + } + // A microflow's documentation is already indexed by the typed extraction; + // the catalog-table pass must not index it a second time. + mf := µflows.Microflow{ + BaseElement: model.BaseElement{ID: "mf-create"}, + ContainerID: modID, + Name: "CreateOrder", + Documentation: note("Microflow notes: ", word), + } + + cat, err := New() + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { cat.Close() }) + tx, err := cat.CatalogDB().Begin() + if err != nil { + t.Fatal(err) + } + b := &Builder{ + catalog: cat, + fullMode: true, + reader: &mock.MockBackend{ + ListModulesFunc: func() ([]*model.Module, error) { + return []*model.Module{{ + BaseElement: model.BaseElement{ID: modID}, + Name: "Sales", + Documentation: note("Module notes: ", word), + }}, nil + }, + ListDomainModelsFunc: func() ([]*domainmodel.DomainModel, error) { return []*domainmodel.DomainModel{dm}, nil }, + ListConstantsFunc: func() ([]*model.Constant, error) { return []*model.Constant{constant}, nil }, + ListPagesFunc: func() ([]*pages.Page, error) { return []*pages.Page{page}, nil }, + ListMicroflowsFunc: func() ([]*microflows.Microflow, error) { return []*microflows.Microflow{mf}, nil }, + ListNanoflowsFunc: func() ([]*microflows.Nanoflow, error) { return nil, nil }, + ListRulesFunc: func() ([]*microflows.Rule, error) { return nil, nil }, + }, + snapshot: &Snapshot{ID: "snap"}, + hierarchy: &hierarchy{ + moduleIDs: map[model.ID]bool{modID: true}, + moduleNames: map[model.ID]string{modID: "Sales"}, + containerParent: map[model.ID]model.ID{"dm-sales": modID}, + folderNames: map[model.ID]string{}, + }, + tx: tx, + } + // In the order Build runs them: strings reads the tables the others fill. + for _, step := range []struct { + name string + build func() error + }{ + {"modules", b.buildModules}, + {"entities", b.buildEntities}, + {"constants", b.buildConstants}, + {"microflows", b.buildMicroflows}, + {"pages", b.buildPages}, + {"strings", b.buildStrings}, + } { + if err := step.build(); err != nil { + t.Fatalf("%s: %v", step.name, err) + } + } + if err := tx.Commit(); err != nil { + t.Fatal(err) + } + return cat +} + +// A word that appears only in the documentation of an entity, an attribute, a +// constant or a page used to be invisible to SEARCH: those fields are plain +// strings in catalog tables, not Texts$Text, so neither the translatable walk +// nor the typed extractions reached them. Each hit must also say which element +// and which kind of field it came from, and which module it belongs to: a +// module's own rows name the module itself, not the empty ModuleName the +// objects view carries for a MODULE, so a filter on ModuleName (excluding +// Marketplace modules, say) can see them. +func TestBuildStrings_IndexesElementDocumentation(t *testing.T) { + cat := docFixture(t, "zebracorn") + + res, err := cat.Query(`SELECT QualifiedName, ObjectType, StringContext, ElementId, ModuleName + FROM strings WHERE strings MATCH 'zebracorn'`) + if err != nil { + t.Fatal(err) + } + type hit struct{ objectType, context, elementID, module string } + // A module has two rows (its own documentation and its domain model's), so + // the key is the element and the field. + got := map[string]hit{} + for _, row := range res.Rows { + qn, _ := row[0].(string) + ot, _ := row[1].(string) + cx, _ := row[2].(string) + id, _ := row[3].(string) + mod, _ := row[4].(string) + key := qn + " / " + cx + if _, dup := got[key]; dup { + t.Errorf("%s indexed more than once", key) + } + got[key] = hit{ot, cx, id, mod} + } + + want := map[string]hit{ + "Sales.Order / documentation": {"ENTITY", "documentation", "ent-order", "Sales"}, + "Sales.Order.Total / documentation": {"ATTRIBUTE", "documentation", "attr-total", "Sales"}, + "Sales.Limit / documentation": {"CONSTANT", "documentation", "const-limit", "Sales"}, + "Sales.Home / documentation": {"PAGE", "documentation", "page-home", "Sales"}, + "Sales.CreateOrder / documentation": {"MICROFLOW", "documentation", "mf-create", "Sales"}, + "Sales / documentation": {"MODULE", "documentation", "mod-sales", "Sales"}, + "Sales / domain_model_documentation": {"MODULE", "domain_model_documentation", "mod-sales", "Sales"}, + } + for key, w := range want { + g, ok := got[key] + if !ok { + t.Errorf("%s: documentation never reached the strings index", key) + continue + } + if g != w { + t.Errorf("%s: got %+v, want %+v", key, g, w) + } + } + for key := range got { + if _, ok := want[key]; !ok { + t.Errorf("unexpected hit %s: %+v", key, got[key]) + } + } +} + +// An element with no documentation must not produce an empty row: an empty +// StringValue is what the insert closure already refuses, and the objects view +// carries ” for every kind that has no documentation field. +func TestBuildStrings_UndocumentedElementsAddNoRows(t *testing.T) { + cat := docFixture(t, "") + res, err := cat.Query(`SELECT QualifiedName, StringValue FROM strings WHERE StringContext IN ('documentation', 'domain_model_documentation')`) + if err != nil { + t.Fatal(err) + } + for _, row := range res.Rows { + v, _ := row[1].(string) + if v == "" { + t.Errorf("empty documentation row for %v", row[0]) + } + } +} + +// note returns prefix+word, or "" for an element that has no documentation. +func note(prefix, word string) string { + if word == "" { + return "" + } + return prefix + word +} + +// A log message is a Microflows$StringTemplate whose Text is a plain string, not +// a Texts$Text, so the translatable walk never sees it: only the log node name +// reached the index, and a word that appears only in the message was invisible +// to SEARCH. +func TestExtractLogNodeNames_IndexesTheLogMessage(t *testing.T) { + act := µflows.ActionActivity{ + Action: µflows.LogMessageAction{ + LogNodeName: "'Orders'", + MessageTemplate: &model.Text{Translations: map[string]string{"en_US": "Order failed: zebracorn"}}, + }, + } + act.ID = "act-log" + oc := µflows.MicroflowObjectCollection{Objects: []microflows.MicroflowObject{act}} + + type row struct{ objType, value, context, elementID, module string } + var got []row + extractLogNodeNames(oc, "Sales.CreateOrder", "MICROFLOW", "Sales", + func(qn, objType, value, ctx, lang, elementID, module string) { + if qn != "Sales.CreateOrder" || lang != "" { + t.Errorf("qn = %q, lang = %q", qn, lang) + } + got = append(got, row{objType, value, ctx, elementID, module}) + }) + + want := []row{ + {"MICROFLOW", "'Orders'", "log_node", "act-log", "Sales"}, + {"MICROFLOW", "Order failed: zebracorn", "log_message", "act-log", "Sales"}, + } + if len(got) != len(want) { + t.Fatalf("rows = %+v, want %+v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Errorf("row %d = %+v, want %+v", i, got[i], want[i]) + } + } +} diff --git a/mdl/catalog/tables.go b/mdl/catalog/tables.go index 32b09e6cc3..d2463fc6b8 100644 --- a/mdl/catalog/tables.go +++ b/mdl/catalog/tables.go @@ -7,6 +7,11 @@ package catalog // // History: // +// 25 (documentation in the strings index): strings gains a row for the +// documentation of every element the catalog reads a Description for +// (entities, attributes, constants, pages, ...), so SEARCH finds words +// that appear only there. Without the bump a cached catalog keeps +// answering "No matches found." for them. // 24 (Studio Pro document names): the views rest_clients, rest_operations, // odata_clients, odata_services and queues are now consumed_rest_services, // consumed_rest_operations, consumed_odata_services, @@ -131,7 +136,7 @@ package catalog // SnapshotSource / SourceId / SourceBranch / SourceRevision columns // from every row (issue #576). // 1 — initial flat schema with denormalized snapshot columns on every row. -const CatalogSchemaVersion = "24" +const CatalogSchemaVersion = "25" // MetaSchemaVersion is the catalog_meta key that records the schema version // the cache was built against.